feat(QTDI-2134): dynamic dependencies

component-api/src/main/java/org/talend/sdk/component/api/configuration/type/DynamicDependenciesServiceConfiguration.java → component-api/src/main/java/org/talend/sdk/component/api/configuration/type/DynamicDependenciesConfiguration.java

@@ -26,9 +26,9 @@
 
 @Target(TYPE)
 @Retention(RUNTIME)
-@ConfigurationType("dynamicDependenciesServiceConfiguration")
-@Documentation("Mark a model (complex object) as being the configuration used in services annotated with @DynamicDependencies.")
-public @interface DynamicDependenciesServiceConfiguration {
+@ConfigurationType("dynamicDependenciesConfiguration")
+@Documentation("Mark a model (complex object) as being the configuration expected to compute dynamic dependencies.")
+public @interface DynamicDependenciesConfiguration {
 
     String value() default "default";
 }

component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/ComponentManager.java

@@ -113,6 +113,7 @@
 import org.apache.xbean.finder.archive.FileArchive;
 import org.apache.xbean.finder.archive.FilteredArchive;
 import org.apache.xbean.finder.archive.JarArchive;
+import org.apache.xbean.finder.filter.ContainsFilter;
 import org.apache.xbean.finder.filter.ExcludeIncludeFilter;
 import org.apache.xbean.finder.filter.Filter;
 import org.apache.xbean.finder.filter.FilterList;
@@ -211,11 +212,14 @@ private static ComponentManager buildNewComponentManager() {
                 {
                     info("ComponentManager version: " + ComponentManagerVersion.VERSION);
                     info("Creating the contextual ComponentManager instance " + getIdentifiers());
-
-                    parallelIf(Boolean.getBoolean("talend.component.manager.plugins.parallel"),
-                            container.getDefinedNestedPlugin().stream().filter(p -> !hasPlugin(p)))
-                            .forEach(this::addPlugin);
-                    info("Components: " + availablePlugins());
+                    try {
+                        parallelIf(Boolean.getBoolean("talend.component.manager.plugins.parallel"),
+                                container.getDefinedNestedPlugin().stream().filter(p -> !hasPlugin(p)))
+                                .forEach(this::addPlugin);
+                        info("Components: " + availablePlugins());
+                    } catch (Exception e) {
+                        info("Failed to load plugins from plugins.properties: " + e.getMessage());
+                    }
                 }
 
                 @Override
@@ -312,6 +316,8 @@ public String[] categories() {
     // + tcomp "runtime" indeed (invisible from the components but required for the runtime
     private final Filter classesFilter;
 
+    private final Filter resourcesFilter;
+
     private final ParameterModelService parameterModelService;
 
     private final InternationalizationServiceFactory internationalizationServiceFactory;
@@ -426,7 +432,12 @@ public ComponentManager(final Path m2, final String dependenciesResource, final
                 .distinct()
                 .map(PrefixFilter::new)
                 .toArray(Filter[]::new));
-
+        resourcesFilter = new FilterList(Stream.concat(
+                Stream.of("META-INF/services/"),
+                additionalParentResources())
+                .distinct()
+                .map(ContainsFilter::new)
+                .toArray(Filter[]::new));
         jsonpProvider = loadJsonProvider();
         jsonbProvider = loadJsonbProvider();
         // these factories have memory caches so ensure we reuse them properly
@@ -460,13 +471,15 @@ public ComponentManager(final Path m2, final String dependenciesResource, final
         migrationHandlerFactory = new MigrationHandlerFactory(reflections);
 
         final Predicate<String> isContainerClass = name -> isContainerClass(classesFilter, name);
+        final Predicate<String> isParentResource = name -> isParentResource(resourcesFilter, name);
         final ContainerManager.ClassLoaderConfiguration defaultClassLoaderConfiguration =
                 ContainerManager.ClassLoaderConfiguration
                         .builder()
                         .parent(tccl)
                         .parentClassesFilter(isContainerClass)
                         .classesFilter(isContainerClass.negate())
                         .supportsResourceDependencies(true)
+                        .parentResourcesFilter(isParentResource)
                         .create();
         this.container = new ContainerManager(ContainerManager.DependenciesResolutionConfiguration
                 .builder()
@@ -611,6 +624,16 @@ private Stream<String> additionalContainerClasses() {
                                 .orElseGet(Stream::empty));
     }
 
+    private Stream<String> additionalParentResources() {
+        return Stream
+                .concat(customizers.stream().flatMap(Customizer::parentResources),
+                        ofNullable(
+                                System.getProperty("talend.component.manager.classloader.container.parentResources"))
+                                .map(s -> s.split(","))
+                                .map(Stream::of)
+                                .orElseGet(Stream::empty));
+    }
+
     public static Path findM2() {
         return new MavenRepositoryDefaultResolver().discover();
     }
@@ -902,13 +925,26 @@ private void autoDiscoverPlugins0(final boolean callers, final boolean classpath
                 final Enumeration<URL> componentMarkers =
                         Thread.currentThread().getContextClassLoader().getResources("TALEND-INF/dependencies.txt");
                 while (componentMarkers.hasMoreElements()) {
-                    File file = Files.toFile(componentMarkers.nextElement());
-                    if (file.getName().equals("dependencies.txt") && file.getParentFile() != null
-                            && file.getParentFile().getName().equals("TALEND-INF")) {
-                        file = file.getParentFile().getParentFile();
-                    }
-                    if (!hasPlugin(container.buildAutoIdFromName(file.getName()))) {
-                        addPlugin(file.getAbsolutePath());
+                    final URL marker = componentMarkers.nextElement();
+                    File file = Files.toFile(marker);
+                    if (file != null) {
+                        if (file.getName().equals("dependencies.txt") && file.getParentFile() != null
+                                && file.getParentFile().getName().equals("TALEND-INF")) {
+                            file = file.getParentFile().getParentFile();
+                        }
+                        if (!hasPlugin(container.buildAutoIdFromName(file.getName()))) {
+                            addPlugin(file.getAbsolutePath());
+                        }
+                    } else {
+                        // lookup nested jar
+                        if (marker != null && "jar".equals(marker.getProtocol())) {
+                            final String urlFile = marker.getFile();
+                            final String jarPath = urlFile.substring(0, urlFile.lastIndexOf("!"));
+                            final String jarFilePath = jarPath.substring(jarPath.lastIndexOf("/") + 1);
+                            if (!hasPlugin(container.buildAutoIdFromName(jarFilePath))) {
+                                addPlugin(jarPath);
+                            }
+                        }
                     }
                 }
             } catch (final IOException e) {
@@ -1045,6 +1081,10 @@ protected boolean isContainerClass(final Filter filter, final String name) {
         return name != null && filter.accept(name);
     }
 
+    protected boolean isParentResource(final Filter filter, final String name) {
+        return name != null && filter.accept(name);
+    }
+
     @Override
     public void close() {
         container.close();
@@ -1722,7 +1762,7 @@ private Archive toArchive(final String module, final OriginalId originalId,
                     return true;
                 }).map(nested -> {
                     if ("nested".equals(nested.getProtocol())
-                            || (nested.getPath() != null && nested.getPath().contains("!/MAVEN-INF/repository/"))) {
+                            || (nested.getPath() != null && nested.getPath().contains(NESTED_MAVEN_REPOSITORY))) {
                         JarInputStream jarStream = null;
                         try {
                             jarStream = new JarInputStream(nested.openStream());
@@ -2207,6 +2247,13 @@ public interface Customizer {
          */
         Stream<String> containerClassesAndPackages();
 
+        /**
+         * @return
+         */
+        default Stream<String> parentResources() {
+            return Stream.empty();
+        }
+
         /**
          * @return advanced toggle to ignore built-in beam exclusions and let this customizer override them.
          */

component-runtime-manager/src/test/java/org/talend/sdk/component/runtime/manager/ComponentManagerTest.java

@@ -441,8 +441,9 @@ void extendFamilyInNestedRepo(@TempDir final File temporaryFolder) throws Except
                         .map(File::getName)
                         .sorted()
                         .toArray(String[]::new);
-                assertEquals(1, dependencies.length); // ignored transitive deps, enables the new root to control it
-                assertEquals("main.jar", dependencies[0]); // transitive-1.0.0.jar is nested
+                assertEquals(2, dependencies.length); // ignored transitive deps, enables the new root to control it
+                assertEquals("fatjar.jar", dependencies[0]);
+                assertEquals("main.jar", dependencies[1]); // transitive-1.0.0.jar is nested
             } finally {
                 if (!transitive.delete()) {
                     transitive.deleteOnExit();

container/container-core/src/main/java/org/talend/sdk/component/classloader/ConfigurableClassLoader.java

@@ -92,6 +92,9 @@ public class ConfigurableClassLoader extends URLClassLoader {
     @Getter
     private final Predicate<String> childFirstFilter;
 
+    @Getter
+    private final Predicate<String> resourcesFilter;
+
     private final Map<String, Collection<Resource>> resources = new HashMap<>();
 
     private final Collection<ClassFileTransformer> transformers = new ArrayList<>();
@@ -111,23 +114,36 @@ public class ConfigurableClassLoader extends URLClassLoader {
     @Getter
     private final List<String> cacheableClasses;
 
+    private List<URL> nestedURLs = new ArrayList<>();
+
     public ConfigurableClassLoader(final String id, final URL[] urls, final ClassLoader parent,
             final Predicate<String> parentFilter, final Predicate<String> childFirstFilter,
             final String[] nestedDependencies, final String[] jvmPrefixes) {
-        this(id, urls, parent, parentFilter, childFirstFilter, emptyMap(), jvmPrefixes);
+        this(id, urls, parent, parentFilter, childFirstFilter, emptyMap(), jvmPrefixes, (name) -> false);
+        if (nestedDependencies != null) {
+            loadNestedDependencies(parent, nestedDependencies);
+        }
+    }
+
+    public ConfigurableClassLoader(final String id, final URL[] urls, final ClassLoader parent,
+            final Predicate<String> parentFilter, final Predicate<String> childFirstFilter,
+            final String[] nestedDependencies, final String[] jvmPrefixes, final Predicate<String> resourcesFilter) {
+        this(id, urls, parent, parentFilter, childFirstFilter, emptyMap(), jvmPrefixes, resourcesFilter);
         if (nestedDependencies != null) {
             loadNestedDependencies(parent, nestedDependencies);
         }
     }
 
     private ConfigurableClassLoader(final String id, final URL[] urls, final ClassLoader parent,
             final Predicate<String> parentFilter, final Predicate<String> childFirstFilter,
-            final Map<String, Collection<Resource>> resources, final String[] jvmPrefixes) {
+            final Map<String, Collection<Resource>> resources, final String[] jvmPrefixes,
+            final Predicate<String> resourcesFilter) {
         super(urls, parent);
         this.id = id;
         this.creationUrls = urls;
         this.parentFilter = parentFilter;
         this.childFirstFilter = childFirstFilter;
+        this.resourcesFilter = resourcesFilter;
         this.resources.putAll(resources);
 
         this.fullPathJvmPrefixes =
@@ -155,6 +171,7 @@ private void loadNestedDependencies(final ClassLoader parent, final String[] nes
             if (url == null) {
                 throw new IllegalArgumentException("Didn't find " + resource + " in " + asList(nestedDependencies));
             }
+            nestedURLs.add(url);
             final Map<String, Resource> resources = new HashMap<>();
             final URLConnection urlConnection;
             final Manifest manifest;
@@ -229,7 +246,7 @@ public void registerTransformer(final ClassFileTransformer transformer) {
     public synchronized URLClassLoader createTemporaryCopy() {
         final ConfigurableClassLoader self = this;
         return temporaryCopy == null ? temporaryCopy = new ConfigurableClassLoader(id, creationUrls, getParent(),
-                parentFilter, childFirstFilter, resources, fullPathJvmPrefixes) {
+                parentFilter, childFirstFilter, resources, fullPathJvmPrefixes, resourcesFilter) {
 
             @Override
             public synchronized void close() throws IOException {
@@ -458,11 +475,26 @@ public Enumeration<URL> findResources(final String name) throws IOException {
         return enumeration(aggregated);
     }
 
+    @Override
+    public URL[] getURLs() {
+        final List<URL> urls = new ArrayList<>(Arrays.asList(super.getURLs()));
+        if (!nestedURLs.isEmpty()) {
+            urls.addAll(nestedURLs);
+        }
+        return urls.toArray(new URL[0]);
+    }
+
     private boolean isNestedDependencyResource(final String name) {
         return name != null && name.startsWith(NESTED_MAVEN_REPOSITORY);
     }
 
     private boolean isInJvm(final URL resource) {
+        // Services and parent allowed resources that should always be found by top level classloader.
+        // By default, META-INF/services/ is always allowed otherwise SPI won't work properly in nested environments.
+        // Warning: selection shouldn't be too generic! Use very specific paths only like jndi.properties.
+        if (resourcesFilter.test(resource.getFile())) {
+            return true;
+        }
         final Path path = toPath(resource);
         if (path == null) {
             return false;

container/container-core/src/main/java/org/talend/sdk/component/container/Container.java

@@ -145,7 +145,8 @@ public Container(final String id, final String rootModule, final Artifact[] depe
                             : null;
             final ConfigurableClassLoader loader = new ConfigurableClassLoader(id, urls,
                     overrideClassLoaderConfig.getParent(), overrideClassLoaderConfig.getParentClassesFilter(),
-                    overrideClassLoaderConfig.getClassesFilter(), rawNestedDependencies, jvmMarkers);
+                    overrideClassLoaderConfig.getClassesFilter(), rawNestedDependencies, jvmMarkers,
+                    overrideClassLoaderConfig.getParentResourcesFilter());
             transformers.forEach(loader::registerTransformer);
             activeSpecificTransformers(loader);
             return loader;
@@ -307,6 +308,10 @@ public Date getCreated() {
         return created.get();
     }
 
+    public boolean hasNestedRepository() {
+        return hasNestedRepository;
+    }
+
     public void registerTransformer(final ClassFileTransformer transformer) {
         transformers.add(transformer);
     }