diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/languageserver/JassDocService.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/languageserver/JassDocService.java index 58059c64d..4ae72e320 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/languageserver/JassDocService.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/languageserver/JassDocService.java @@ -13,7 +13,12 @@ import java.io.IOException; import java.io.InputStream; import java.net.HttpURLConnection; +import java.net.InetSocketAddress; +import java.net.Proxy; +import java.net.URI; import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; @@ -24,15 +29,19 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; +import java.util.Base64; import java.util.Comparator; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; @@ -40,6 +49,10 @@ public final class JassDocService { + // Optional JassDoc configuration (environment variable or system property): + // WURST_JASSDOC_DB_AUTO_UPDATE=false keeps an existing latest DB indefinitely. + // WURST_JASSDOC_DB_PROXY overrides HTTPS_PROXY/HTTP_PROXY for JassDoc requests. + public enum SymbolKind { FUNCTION, VARIABLE } @@ -247,7 +260,7 @@ private Optional lookupDocumentation(LookupKey key) { } try (Connection conn = open(dbPath.get())) { List schemas = discoverSchemas(conn); - boolean hasLegacySchema = hasLegacyJassdocSchema(conn); + boolean hasLegacySchema = hasCompatibleLegacyJassdocSchema(conn); if (schemas.isEmpty() && !hasLegacySchema) { WLogger.warning("JassDoc DB found, but no compatible documentation tables were detected."); initFailed = true; @@ -355,7 +368,7 @@ private void triggerAsyncInit() { } private @Nullable String lookupFromLegacyJassdocTables(Connection conn, LookupKey key) throws SQLException { - if (!tableExists(conn, "parameters")) { + if (!hasCompatibleLegacyJassdocSchema(conn)) { return null; } Map params = readKeyValueRows(conn, "parameters", "fnname", "param", "value", key.symbolName()); @@ -527,7 +540,7 @@ private Optional ensureDbAvailable() throws IOException { } boolean needsDownload = !Files.exists(dbPath); - if (!needsDownload && "latest".equals(revision)) { + if (!needsDownload && "latest".equals(revision) && autoUpdateEnabled()) { needsDownload = isStaleLatest(dbPath); } @@ -590,7 +603,7 @@ private List resolveLatestReleaseAssetUrls() { private List readNewestReleaseAssetUrlsFromList() { List urls = new ArrayList<>(); try { - HttpURLConnection con = (HttpURLConnection) new URL(RELEASES_API).openConnection(); + HttpURLConnection con = openHttpConnection(new URL(RELEASES_API)); con.setConnectTimeout(10_000); con.setReadTimeout(20_000); con.setRequestMethod("GET"); @@ -625,7 +638,7 @@ private List readNewestReleaseAssetUrlsFromList() { private List readReleaseAssetUrls(String apiUrl) { List urls = new ArrayList<>(); try { - HttpURLConnection con = (HttpURLConnection) new URL(apiUrl).openConnection(); + HttpURLConnection con = openHttpConnection(new URL(apiUrl)); con.setConnectTimeout(10_000); con.setReadTimeout(20_000); con.setRequestMethod("GET"); @@ -710,6 +723,14 @@ private boolean isStaleLatest(Path dbPath) throws IOException { return modified.toInstant().isBefore(cutoff); } + boolean autoUpdateEnabled() { + return Utils.getEnvOrConfig("WURST_JASSDOC_DB_AUTO_UPDATE") + .map(value -> !value.equalsIgnoreCase("false") + && !value.equalsIgnoreCase("no") + && !value.equals("0")) + .orElse(true); + } + private Duration parseDurationOrDefault(String text) { try { return Duration.parse(text); @@ -720,22 +741,191 @@ private Duration parseDurationOrDefault(String text) { private void download(String urlString, Path target) throws IOException { URL url = new URL(urlString); - HttpURLConnection con = (HttpURLConnection) url.openConnection(); + HttpURLConnection con = openHttpConnection(url); con.setConnectTimeout(10_000); con.setReadTimeout(20_000); con.setInstanceFollowRedirects(true); con.setRequestMethod("GET"); + con.setRequestProperty("User-Agent", "WurstScript-LSP"); int code = con.getResponseCode(); if (code < 200 || code >= 300) { throw new IOException("HTTP " + code + " for " + urlString); } Path tmp = Files.createTempFile(target.getParent(), "jassdoc-", ".tmp"); - try (InputStream in = con.getInputStream()) { - Files.copy(in, tmp, StandardCopyOption.REPLACE_EXISTING); + try { + try (InputStream in = con.getInputStream()) { + Files.copy(in, tmp, StandardCopyOption.REPLACE_EXISTING); + } + installDownloadedDatabase(tmp, target); } finally { con.disconnect(); + Files.deleteIfExists(tmp); } - Files.move(tmp, target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + } + + void installDownloadedDatabase(Path downloaded, Path target) throws IOException { + validateDownloadedDatabase(downloaded); + + Path backup = target.resolveSibling(target.getFileName() + ".bak"); + if (Files.exists(target)) { + Path backupTmp = Files.createTempFile(target.getParent(), "jassdoc-backup-", ".tmp"); + try { + Files.copy(target, backupTmp, StandardCopyOption.REPLACE_EXISTING, + StandardCopyOption.COPY_ATTRIBUTES); + replaceAtomically(backupTmp, backup); + } finally { + Files.deleteIfExists(backupTmp); + } + } + + try { + replaceAtomically(downloaded, target); + } catch (IOException installFailure) { + if (Files.exists(backup)) { + restoreBackup(backup, target, installFailure); + } + throw installFailure; + } + } + + void restoreBackup(Path backup, Path target, IOException installFailure) { + Path restoreTmp = null; + try { + restoreTmp = Files.createTempFile(target.getParent(), "jassdoc-restore-", ".tmp"); + Files.copy(backup, restoreTmp, StandardCopyOption.REPLACE_EXISTING, + StandardCopyOption.COPY_ATTRIBUTES); + replaceAtomically(restoreTmp, target); + } catch (IOException restoreFailure) { + installFailure.addSuppressed(restoreFailure); + } finally { + if (restoreTmp != null) { + try { + Files.deleteIfExists(restoreTmp); + } catch (IOException cleanupFailure) { + installFailure.addSuppressed(cleanupFailure); + } + } + } + } + + private void validateDownloadedDatabase(Path downloaded) throws IOException { + try (Connection conn = open(downloaded)) { + if (!passesIntegrityCheck(conn)) { + throw new IOException("Downloaded JassDoc database failed SQLite integrity check"); + } + if (discoverSchemas(conn).isEmpty() && !hasCompatibleLegacyJassdocSchema(conn)) { + throw new IOException("Downloaded file is not a compatible JassDoc database"); + } + } catch (SQLException e) { + throw new IOException("Downloaded file is not a valid JassDoc database", e); + } + } + + private boolean passesIntegrityCheck(Connection conn) throws SQLException { + try (Statement statement = conn.createStatement(); + ResultSet result = statement.executeQuery("PRAGMA integrity_check")) { + return result.next() + && "ok".equalsIgnoreCase(result.getString(1)) + && !result.next(); + } + } + + private void replaceAtomically(Path source, Path target) throws IOException { + try { + Files.move(source, target, StandardCopyOption.REPLACE_EXISTING, + StandardCopyOption.ATOMIC_MOVE); + } catch (AtomicMoveNotSupportedException e) { + Files.move(source, target, StandardCopyOption.REPLACE_EXISTING); + } + } + + private HttpURLConnection openHttpConnection(URL url) throws IOException { + Optional proxySetting = selectProxySetting(url, Utils::getEnvOrConfig); + if (proxySetting.isEmpty()) { + return (HttpURLConnection) url.openConnection(); + } + + URI proxyUri = parseHttpProxyUri(proxySetting.get()); + int port = proxyUri.getPort() >= 0 ? proxyUri.getPort() : 80; + Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyUri.getHost(), port)); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(proxy); + if (proxyUri.getUserInfo() != null) { + String credentials = Base64.getEncoder().encodeToString( + proxyUri.getUserInfo().getBytes(StandardCharsets.UTF_8)); + connection.setRequestProperty("Proxy-Authorization", "Basic " + credentials); + } + return connection; + } + + static Optional selectProxySetting(URL url, + Function> lookup) { + Optional proxySetting = lookup.apply("WURST_JASSDOC_DB_PROXY"); + if (proxySetting.isPresent()) { + return proxySetting; + } + Optional noProxy = firstConfigured(lookup, "NO_PROXY", "no_proxy"); + if (noProxy.isPresent() && shouldBypassProxy(url.getHost(), noProxy.get())) { + return Optional.empty(); + } + if ("https".equalsIgnoreCase(url.getProtocol())) { + return firstConfigured(lookup, "HTTPS_PROXY", "https_proxy"); + } + if ("http".equalsIgnoreCase(url.getProtocol())) { + return firstConfigured(lookup, "HTTP_PROXY", "http_proxy"); + } + return Optional.empty(); + } + + static URI parseHttpProxyUri(String value) throws IOException { + URI proxyUri; + try { + proxyUri = URI.create(value.contains("://") ? value : "http://" + value); + } catch (IllegalArgumentException e) { + throw new IOException("Invalid JassDoc proxy URL", e); + } + if (proxyUri.getHost() == null) { + throw new IOException("Invalid JassDoc proxy URL: missing host"); + } + if (!"http".equalsIgnoreCase(proxyUri.getScheme())) { + throw new IOException("Unsupported JassDoc proxy scheme '" + proxyUri.getScheme() + + "'; use an http:// proxy URL"); + } + return proxyUri; + } + + static boolean shouldBypassProxy(String host, String noProxySetting) { + String normalizedHost = host.toLowerCase(Locale.ROOT); + for (String rawEntry : noProxySetting.split(",")) { + String entry = rawEntry.trim().toLowerCase(Locale.ROOT); + if (entry.equals("*")) { + return true; + } + int portSeparator = entry.lastIndexOf(':'); + if (portSeparator > 0 && entry.indexOf(':') == portSeparator) { + entry = entry.substring(0, portSeparator); + } + if (entry.startsWith("*.")) { + entry = entry.substring(1); + } + if (entry.startsWith(".")) { + if (normalizedHost.endsWith(entry) + || normalizedHost.equals(entry.substring(1))) { + return true; + } + } else if (!entry.isEmpty() && (normalizedHost.equals(entry) + || normalizedHost.endsWith("." + entry))) { + return true; + } + } + return false; + } + + private static Optional firstConfigured( + Function> lookup, String... names) { + return Stream.of(names) + .map(lookup) + .flatMap(Optional::stream) + .findFirst(); } private Connection open(Path dbPath) throws SQLException { @@ -778,8 +968,32 @@ private List discoverSchemas(Connection conn) throws SQLException { return result; } - private boolean hasLegacyJassdocSchema(Connection conn) throws SQLException { - return tableExists(conn, "parameters"); + private boolean hasCompatibleLegacyJassdocSchema(Connection conn) throws SQLException { + return tableHasColumns(conn, "parameters", "fnname", "param", "value") + && (!tableExists(conn, "annotations") + || tableHasColumns(conn, "annotations", "fnname", "anname", "value")) + && (!tableExists(conn, "params_extra") + || tableHasColumns(conn, "params_extra", "fnname", "param", "anname", "value")); + } + + private boolean tableHasColumns(Connection conn, String tableName, String... requiredColumns) + throws SQLException { + Set columns = new HashSet<>(); + DatabaseMetaData md = conn.getMetaData(); + try (ResultSet result = md.getColumns(null, null, tableName, "%")) { + while (result.next()) { + String name = result.getString("COLUMN_NAME"); + if (name != null) { + columns.add(name.toLowerCase(Locale.ROOT)); + } + } + } + for (String required : requiredColumns) { + if (!columns.contains(required)) { + return false; + } + } + return true; } private boolean tableExists(Connection conn, String tableName) throws SQLException { diff --git a/de.peeeq.wurstscript/src/test/java/de/peeeq/wurstio/languageserver/JassDocServiceTests.java b/de.peeeq.wurstscript/src/test/java/de/peeeq/wurstio/languageserver/JassDocServiceTests.java new file mode 100644 index 000000000..cea6708fe --- /dev/null +++ b/de.peeeq.wurstscript/src/test/java/de/peeeq/wurstio/languageserver/JassDocServiceTests.java @@ -0,0 +1,181 @@ +package de.peeeq.wurstio.languageserver; + +import org.testng.annotations.Test; + +import java.io.IOException; +import java.io.RandomAccessFile; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Map; +import java.util.Optional; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertThrows; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.expectThrows; + +public class JassDocServiceTests { + + @Test + public void invalidDownloadDoesNotReplaceExistingDatabase() throws IOException { + Path dir = Files.createTempDirectory("jassdoc-invalid-download-"); + Path target = dir.resolve("jassdoc-latest.db"); + Path downloaded = dir.resolve("download.tmp"); + Files.writeString(target, "working database", StandardCharsets.UTF_8); + Files.writeString(downloaded, "proxy error page", StandardCharsets.UTF_8); + + assertThrows(IOException.class, + () -> new JassDocService().installDownloadedDatabase(downloaded, target)); + + assertEquals(Files.readString(target, StandardCharsets.UTF_8), "working database"); + assertFalse(Files.exists(dir.resolve("jassdoc-latest.db.bak"))); + } + + @Test + public void successfulUpdateKeepsPreviousDatabaseBackup() throws Exception { + Path dir = Files.createTempDirectory("jassdoc-valid-download-"); + Path target = dir.resolve("jassdoc-latest.db"); + Path downloaded = dir.resolve("download.tmp"); + Files.writeString(target, "previous database", StandardCharsets.UTF_8); + try (Connection conn = DriverManager.getConnection("jdbc:sqlite:" + downloaded.toAbsolutePath()); + Statement statement = conn.createStatement()) { + statement.execute("CREATE TABLE docs(name TEXT, documentation TEXT)"); + statement.execute("INSERT INTO docs VALUES ('GetUnitX', 'Returns the x coordinate')"); + } + + new JassDocService().installDownloadedDatabase(downloaded, target); + + assertEquals(Files.readString(dir.resolve("jassdoc-latest.db.bak"), StandardCharsets.UTF_8), + "previous database"); + try (Connection conn = DriverManager.getConnection("jdbc:sqlite:" + target.toAbsolutePath()); + Statement statement = conn.createStatement(); + ResultSet result = statement.executeQuery("SELECT documentation FROM docs WHERE name = 'GetUnitX'")) { + assertTrue(result.next()); + assertEquals(result.getString(1), "Returns the x coordinate"); + } + } + + @Test + public void incompleteLegacySchemaDoesNotReplaceExistingDatabase() throws Exception { + Path dir = Files.createTempDirectory("jassdoc-incomplete-legacy-"); + Path target = dir.resolve("jassdoc-latest.db"); + Path downloaded = dir.resolve("download.tmp"); + Files.writeString(target, "working database", StandardCharsets.UTF_8); + try (Connection conn = DriverManager.getConnection("jdbc:sqlite:" + downloaded.toAbsolutePath()); + Statement statement = conn.createStatement()) { + statement.execute("CREATE TABLE parameters(fnname TEXT)"); + } + + assertThrows(IOException.class, + () -> new JassDocService().installDownloadedDatabase(downloaded, target)); + assertEquals(Files.readString(target, StandardCharsets.UTF_8), "working database"); + } + + @Test + public void corruptDatabaseWithValidSchemaDoesNotReplaceExistingDatabase() throws Exception { + Path dir = Files.createTempDirectory("jassdoc-corrupt-download-"); + Path target = dir.resolve("jassdoc-latest.db"); + Path downloaded = dir.resolve("download.tmp"); + Files.writeString(target, "working database", StandardCharsets.UTF_8); + int pageSize; + int indexRootPage; + try (Connection conn = DriverManager.getConnection("jdbc:sqlite:" + downloaded.toAbsolutePath()); + Statement statement = conn.createStatement()) { + statement.execute("CREATE TABLE docs(name TEXT, documentation TEXT)"); + for (int i = 0; i < 1_000; i++) { + statement.execute("INSERT INTO docs VALUES ('name" + i + "', '" + + "documentation".repeat(40) + "')"); + } + statement.execute("CREATE INDEX docs_name_idx ON docs(name)"); + try (ResultSet result = statement.executeQuery("PRAGMA page_size")) { + assertTrue(result.next()); + pageSize = result.getInt(1); + } + try (ResultSet result = statement.executeQuery( + "SELECT rootpage FROM sqlite_master WHERE name = 'docs_name_idx'")) { + assertTrue(result.next()); + indexRootPage = result.getInt(1); + } + } + try (RandomAccessFile file = new RandomAccessFile(downloaded.toFile(), "rw")) { + file.seek((long) (indexRootPage - 1) * pageSize); + file.write(new byte[32]); + } + + assertThrows(IOException.class, + () -> new JassDocService().installDownloadedDatabase(downloaded, target)); + assertEquals(Files.readString(target, StandardCharsets.UTF_8), "working database"); + } + + @Test + public void restoreOverwritesAResidualPartialTarget() throws IOException { + Path dir = Files.createTempDirectory("jassdoc-restore-"); + Path target = dir.resolve("jassdoc-latest.db"); + Path backup = dir.resolve("jassdoc-latest.db.bak"); + Files.writeString(target, "partial replacement", StandardCharsets.UTF_8); + Files.writeString(backup, "working database", StandardCharsets.UTF_8); + IOException installFailure = new IOException("simulated interrupted replacement"); + + new JassDocService().restoreBackup(backup, target, installFailure); + + assertEquals(Files.readString(target, StandardCharsets.UTF_8), "working database"); + assertEquals(installFailure.getSuppressed().length, 0); + } + + @Test + public void automaticUpdatesCanBeDisabled() { + String previous = System.getProperty("WURST_JASSDOC_DB_AUTO_UPDATE"); + try { + System.setProperty("WURST_JASSDOC_DB_AUTO_UPDATE", "false"); + assertFalse(new JassDocService().autoUpdateEnabled()); + } finally { + if (previous == null) { + System.clearProperty("WURST_JASSDOC_DB_AUTO_UPDATE"); + } else { + System.setProperty("WURST_JASSDOC_DB_AUTO_UPDATE", previous); + } + } + } + + @Test + public void proxyBypassSupportsStandardHostForms() { + assertTrue(JassDocService.shouldBypassProxy( + "api.github.com", "localhost, .github.com, internal.example:8080")); + assertTrue(JassDocService.shouldBypassProxy( + "github.com", "localhost, .github.com, internal.example:8080")); + assertTrue(JassDocService.shouldBypassProxy( + "internal.example", "localhost, github.com, internal.example:8080")); + assertFalse(JassDocService.shouldBypassProxy( + "github.com", "localhost, example.com")); + } + + @Test + public void standardProxyMatchesRequestProtocol() throws Exception { + Map settings = Map.of( + "HTTPS_PROXY", "http://secure-proxy.example:8443", + "HTTP_PROXY", "http://plain-proxy.example:8080"); + + assertEquals(JassDocService.selectProxySetting( + URI.create("https://github.com/example").toURL(), + name -> Optional.ofNullable(settings.get(name))).orElseThrow(), + "http://secure-proxy.example:8443"); + assertEquals(JassDocService.selectProxySetting( + URI.create("http://mirror.example/jass.db").toURL(), + name -> Optional.ofNullable(settings.get(name))).orElseThrow(), + "http://plain-proxy.example:8080"); + } + + @Test + public void unsupportedTlsProxyIsRejectedExplicitly() { + IOException error = expectThrows(IOException.class, + () -> JassDocService.parseHttpProxyUri("https://proxy.example")); + assertTrue(error.getMessage().contains("use an http:// proxy URL")); + } +}