diff --git a/api/src/main/java/org/commonjava/maven/galley/model/Transfer.java b/api/src/main/java/org/commonjava/maven/galley/model/Transfer.java index e6943d4dd..00dffe889 100644 --- a/api/src/main/java/org/commonjava/maven/galley/model/Transfer.java +++ b/api/src/main/java/org/commonjava/maven/galley/model/Transfer.java @@ -33,6 +33,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.util.Set; import static org.apache.commons.lang.StringUtils.join; @@ -518,6 +519,21 @@ public void setResource( ConcreteResource resource ) this.resource = resource; } + public Set getProxySitesCache() + { + return provider.getProxySitesCache(); + } + + public boolean isProxySite( String site ) + { + return provider.isProxySite( site ); + } + + public void saveProxySite( String site ) + { + provider.saveProxySite( site ); + } + @Override public boolean equals( final Object o ) { diff --git a/api/src/main/java/org/commonjava/maven/galley/spi/cache/CacheProvider.java b/api/src/main/java/org/commonjava/maven/galley/spi/cache/CacheProvider.java index 2292790a9..d96769d87 100644 --- a/api/src/main/java/org/commonjava/maven/galley/spi/cache/CacheProvider.java +++ b/api/src/main/java/org/commonjava/maven/galley/spi/cache/CacheProvider.java @@ -124,17 +124,31 @@ void createAlias( ConcreteResource from, ConcreteResource to ) AdminView asAdminView (); + Set getProxySitesCache(); + + boolean isProxySite( String site ); + + void saveProxySite( String site ); + interface AdminView extends CacheProvider { boolean isFileBased(); - default File getDetachedFile( ConcreteResource resource ) { return null; } + default File getDetachedFile( ConcreteResource resource ) + { + return null; + } /** * Expose GC in case the caller wants to run it directly, especially in functional test */ - default void gc() {} + default void gc() + { + } - default void close() {} + default void close() + { + } } + } diff --git a/caches/partyline/src/main/java/org/commonjava/maven/galley/cache/partyline/PartyLineCacheProvider.java b/caches/partyline/src/main/java/org/commonjava/maven/galley/cache/partyline/PartyLineCacheProvider.java index 78032e5a9..fcfd94a1b 100644 --- a/caches/partyline/src/main/java/org/commonjava/maven/galley/cache/partyline/PartyLineCacheProvider.java +++ b/caches/partyline/src/main/java/org/commonjava/maven/galley/cache/partyline/PartyLineCacheProvider.java @@ -41,6 +41,7 @@ import java.util.Iterator; import java.util.List; import java.util.Objects; +import java.util.Set; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadPoolExecutor; @@ -515,4 +516,21 @@ public PartyLineCacheProviderConfig getConfig() { return config; } + + @Override + public Set getProxySitesCache() + { + return null; + } + + @Override + public boolean isProxySite( String site ) + { + return false; + } + + @Override + public void saveProxySite( String site ) + { + } } diff --git a/caches/path-mapped/src/main/java/org/commonjava/maven/galley/cache/pathmapped/PathMappedCacheProvider.java b/caches/path-mapped/src/main/java/org/commonjava/maven/galley/cache/pathmapped/PathMappedCacheProvider.java index 537549ce2..e0176f2b7 100644 --- a/caches/path-mapped/src/main/java/org/commonjava/maven/galley/cache/pathmapped/PathMappedCacheProvider.java +++ b/caches/path-mapped/src/main/java/org/commonjava/maven/galley/cache/pathmapped/PathMappedCacheProvider.java @@ -15,17 +15,17 @@ */ package org.commonjava.maven.galley.cache.pathmapped; -import org.commonjava.maven.galley.model.SpecialPathInfo; -import org.commonjava.maven.galley.spi.io.SpecialPathManager; -import org.commonjava.maven.galley.util.PathUtils; -import org.commonjava.storage.pathmapped.core.PathMappedFileManager; import org.commonjava.maven.galley.io.TransferDecoratorManager; import org.commonjava.maven.galley.model.ConcreteResource; import org.commonjava.maven.galley.model.Location; +import org.commonjava.maven.galley.model.SpecialPathInfo; import org.commonjava.maven.galley.model.Transfer; import org.commonjava.maven.galley.spi.cache.CacheProvider; import org.commonjava.maven.galley.spi.event.FileEventManager; import org.commonjava.maven.galley.spi.io.PathGenerator; +import org.commonjava.maven.galley.spi.io.SpecialPathManager; +import org.commonjava.maven.galley.util.PathUtils; +import org.commonjava.storage.pathmapped.core.PathMappedFileManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,6 +36,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.util.Arrays; +import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; @@ -409,6 +410,24 @@ public void stopReporting() { } + @Override + public Set getProxySitesCache() + { + return fileManager.getProxySitesCache(); + } + + @Override + public boolean isProxySite( final String site ) + { + return fileManager.isProxySite( site ); + } + + @Override + public void saveProxySite( String site ) + { + fileManager.saveProxySite( site ); + } + public PathMappedFileManager getPathMappedFileManager() { return fileManager; diff --git a/caches/path-mapped/src/test/resources/META-INF/persistence.xml b/caches/path-mapped/src/test/resources/META-INF/persistence.xml index 80717d1a8..83769cc31 100644 --- a/caches/path-mapped/src/test/resources/META-INF/persistence.xml +++ b/caches/path-mapped/src/test/resources/META-INF/persistence.xml @@ -10,6 +10,7 @@ org.commonjava.storage.pathmapped.pathdb.jpa.model.JpaReverseKey org.commonjava.storage.pathmapped.pathdb.jpa.model.JpaReverseMap org.commonjava.storage.pathmapped.pathdb.jpa.model.JpaReclaim + org.commonjava.storage.pathmapped.pathdb.jpa.model.JpaProxySite diff --git a/core/src/main/java/org/commonjava/maven/galley/cache/FileCacheProvider.java b/core/src/main/java/org/commonjava/maven/galley/cache/FileCacheProvider.java index 49339d873..148947794 100644 --- a/core/src/main/java/org/commonjava/maven/galley/cache/FileCacheProvider.java +++ b/core/src/main/java/org/commonjava/maven/galley/cache/FileCacheProvider.java @@ -40,6 +40,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; @@ -442,4 +443,21 @@ public void close() } } } + + @Override + public Set getProxySitesCache() + { + return null; + } + + @Override + public boolean isProxySite( String site ) + { + return false; + } + + @Override + public void saveProxySite( String site ) + { + } } diff --git a/pom.xml b/pom.xml index 3c4b8aecd..a96e65758 100644 --- a/pom.xml +++ b/pom.xml @@ -52,7 +52,7 @@ https://github.com/Commonjava/galley 1.1.3 1.16 - 2.5 + 3.0-SNAPSHOT 1.24 1.14 9.4.7.Final diff --git a/testing/api/src/main/java/org/commonjava/maven/galley/testing/core/cache/TestCacheProvider.java b/testing/api/src/main/java/org/commonjava/maven/galley/testing/core/cache/TestCacheProvider.java index 76fc86fc5..7ce03ee72 100644 --- a/testing/api/src/main/java/org/commonjava/maven/galley/testing/core/cache/TestCacheProvider.java +++ b/testing/api/src/main/java/org/commonjava/maven/galley/testing/core/cache/TestCacheProvider.java @@ -15,14 +15,6 @@ */ package org.commonjava.maven.galley.testing.core.cache; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.file.Files; - import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.commonjava.maven.galley.cache.SimpleLockingSupport; @@ -33,6 +25,13 @@ import org.commonjava.maven.galley.spi.cache.CacheProvider; import org.commonjava.maven.galley.spi.event.FileEventManager; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.util.Set; + public class TestCacheProvider implements CacheProvider { @@ -300,4 +299,20 @@ public void stopReporting() lockingSupport.stopReporting(); } + @Override + public Set getProxySitesCache() + { + return null; + } + + @Override + public boolean isProxySite( String site ) + { + return false; + } + + @Override + public void saveProxySite( String site ) + { + } } diff --git a/transports/httpclient/src/main/java/org/commonjava/maven/galley/transport/htcli/internal/AbstractHttpJob.java b/transports/httpclient/src/main/java/org/commonjava/maven/galley/transport/htcli/internal/AbstractHttpJob.java index a542920e8..68ea07d03 100644 --- a/transports/httpclient/src/main/java/org/commonjava/maven/galley/transport/htcli/internal/AbstractHttpJob.java +++ b/transports/httpclient/src/main/java/org/commonjava/maven/galley/transport/htcli/internal/AbstractHttpJob.java @@ -47,6 +47,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Set; import static org.commonjava.o11yphant.trace.TraceManager.addFieldToActiveSpan; @@ -104,6 +105,7 @@ protected boolean executeHttp() { int tries = 1; boolean doProxy = false; + String site = location.getHost(); try { while ( tries > 0 ) @@ -111,6 +113,17 @@ protected boolean executeHttp() tries--; try { + // if the site is in memory, go with proxy directly, avoid to query db frequently through isProxySite + if ( getProxySitesCache().contains( site ) ) + { + doProxy = true; + } + // if the site is not in memory, query db through isProxySite + else if ( isProxySite( site ) ) + { + doProxy = true; + getProxySitesCache().add( site ); // add the site into memory + } client = http.createClient( location, doProxy ); response = client.execute( request, http.createContext( location ) ); @@ -144,6 +157,8 @@ else if ( !doProxy ) // never do with proxy, retry with proxy { tries = 1; doProxy = true; + getProxySitesCache().add( site ); // add the site into memory + saveProxySite( site ); // add the site into db logger.debug( "Retry to execute with global proxy for {}", url ); } else // already did proxy, still timeout @@ -151,8 +166,8 @@ else if ( !doProxy ) // never do with proxy, retry with proxy addFieldToActiveSpan( "target-error-reason", "timeout" ); addFieldToActiveSpan( "target-error", e.getClass().getSimpleName() ); throw new TransferTimeoutException( location, url, - "Repository remote request failed for: {}. Reason: {}", e, - url, e.getMessage() ); + "Retried with proxy, repository remote request timeout failed for: {}. Reason: {}", + e, url, e.getMessage() ); } } catch ( final IOException e ) @@ -160,8 +175,8 @@ else if ( !doProxy ) // never do with proxy, retry with proxy addFieldToActiveSpan( "target-error-reason", "I/O" ); addFieldToActiveSpan( "target-error", e.getClass().getSimpleName() ); throw new TransferLocationException( location, - "Repository remote request failed for: {}. Reason: {}", e, url, - e.getMessage() ); + "Repository remote request IO failed for: {}. Reason: {}", e, + url, e.getMessage() ); } catch ( TransferLocationException e ) { @@ -173,7 +188,7 @@ else if ( !doProxy ) // never do with proxy, retry with proxy { addFieldToActiveSpan( "target-error-reason", "unknown" ); addFieldToActiveSpan( "target-error", e.getClass().getSimpleName() ); - throw new TransferException( "Repository remote request failed for: {}. Reason: {}", e, url, + throw new TransferException( "Repository remote request Galley failed for: {}. Reason: {}", e, url, e.getMessage() ); } } @@ -200,7 +215,6 @@ else if ( !doProxy ) // never do with proxy, retry with proxy } /* for GET/HEAD request, need to override below two methods for writeMetadata() */ - protected Transfer getTransfer() { return null; @@ -307,4 +321,18 @@ protected void cleanup() response = null; } + private boolean isProxySite( String site ) + { + return getTransfer().isProxySite( site ); + } + + private void saveProxySite( String site ) + { + getTransfer().saveProxySite( site ); + } + + private Set getProxySitesCache() + { + return getTransfer().getProxySitesCache(); + } } diff --git a/transports/httpclient/src/test/java/org/commonjava/maven/galley/transport/htcli/testutil/TestCacheProvider.java b/transports/httpclient/src/test/java/org/commonjava/maven/galley/transport/htcli/testutil/TestCacheProvider.java index ae4a34606..1ebe35577 100644 --- a/transports/httpclient/src/test/java/org/commonjava/maven/galley/transport/htcli/testutil/TestCacheProvider.java +++ b/transports/httpclient/src/test/java/org/commonjava/maven/galley/transport/htcli/testutil/TestCacheProvider.java @@ -15,14 +15,6 @@ */ package org.commonjava.maven.galley.transport.htcli.testutil; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.file.Files; - import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.commonjava.maven.galley.cache.SimpleLockingSupport; @@ -33,6 +25,13 @@ import org.commonjava.maven.galley.spi.cache.CacheProvider; import org.commonjava.maven.galley.spi.event.FileEventManager; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.util.Set; + public class TestCacheProvider implements CacheProvider { @@ -299,4 +298,21 @@ public void stopReporting() { lockingSupport.stopReporting(); } + + @Override + public Set getProxySitesCache() + { + return null; + } + + @Override + public boolean isProxySite( String site ) + { + return false; + } + + @Override + public void saveProxySite( String site ) + { + } }