From 607aedc60436c739f0a0fce23b4830c7e1561a53 Mon Sep 17 00:00:00 2001 From: gotty Date: Tue, 14 Jan 2020 00:55:52 +0300 Subject: [PATCH] [LIB-13] Add query params to settings --- .../java/org/hedgecode/chess/scanner/Settings.java | 4 + .../hedgecode/chess/scanner/json/JSONSettings.java | 16 ++ .../scanner/portal/AbstractRequestScanner.java | 193 +++++++-------------- .../scanner/portal/AbstractSettingsScanner.java | 22 ++- .../chess/scanner/portal/ChessGamesScanner.java | 15 +- 5 files changed, 99 insertions(+), 151 deletions(-) diff --git a/src/main/java/org/hedgecode/chess/scanner/Settings.java b/src/main/java/org/hedgecode/chess/scanner/Settings.java index 28bd2c9..e2a6bf9 100644 --- a/src/main/java/org/hedgecode/chess/scanner/Settings.java +++ b/src/main/java/org/hedgecode/chess/scanner/Settings.java @@ -27,6 +27,8 @@ public interface Settings { boolean isTournamentMultiPage(); + String getTournamentQueryParams(); + String getTournamentGamesUrlRegex(); String getTournamentQuery(); @@ -37,4 +39,6 @@ public interface Settings { String getGamePgnUrl(); + String getGameQueryParams(); + } diff --git a/src/main/java/org/hedgecode/chess/scanner/json/JSONSettings.java b/src/main/java/org/hedgecode/chess/scanner/json/JSONSettings.java index ba7b4c0..d4fa8cc 100644 --- a/src/main/java/org/hedgecode/chess/scanner/json/JSONSettings.java +++ b/src/main/java/org/hedgecode/chess/scanner/json/JSONSettings.java @@ -33,6 +33,9 @@ public class JSONSettings implements Settings { @SerializedName("tournamentIsMultiPage") private boolean tournamentIsMultiPage; + @SerializedName("tournamentQueryParams") + private String tournamentQueryParams; + @SerializedName("tournamentGamesUrlRegex") private String tournamentGamesUrlRegex; @@ -48,6 +51,9 @@ public class JSONSettings implements Settings { @SerializedName("gamePgnUrl") private String gamePgnUrl; + @SerializedName("gameQueryParams") + private String gameQueryParams; + @Override public String getTournamentUrl() { return tournamentUrl; @@ -59,6 +65,11 @@ public class JSONSettings implements Settings { } @Override + public String getTournamentQueryParams() { + return tournamentQueryParams; + } + + @Override public String getTournamentGamesUrlRegex() { return tournamentGamesUrlRegex; } @@ -83,4 +94,9 @@ public class JSONSettings implements Settings { return gamePgnUrl; } + @Override + public String getGameQueryParams() { + return gameQueryParams; + } + } diff --git a/src/main/java/org/hedgecode/chess/scanner/portal/AbstractRequestScanner.java b/src/main/java/org/hedgecode/chess/scanner/portal/AbstractRequestScanner.java index fd72e6e..a82422e 100644 --- a/src/main/java/org/hedgecode/chess/scanner/portal/AbstractRequestScanner.java +++ b/src/main/java/org/hedgecode/chess/scanner/portal/AbstractRequestScanner.java @@ -19,8 +19,6 @@ package org.hedgecode.chess.scanner.portal; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; import java.util.List; import java.util.Map; @@ -33,8 +31,9 @@ import org.hedgecode.chess.scanner.ChessHogScannerConstants; import org.hedgecode.chess.scanner.ChessHogScannerException; import org.hedgecode.chess.scanner.Scanner; import org.hedgecode.chess.scanner.regex.RegexMatcher; -import org.hedgecode.chess.scanner.regex.RegexMatcherResult; import org.hedgecode.chess.scanner.regex.RegexResult; +import org.hedgecode.chess.scanner.regex.RegexType; +import org.hedgecode.chess.scanner.regex.RegexTypeMatcher; /** * AbstractRequestScanner @@ -78,157 +77,98 @@ public abstract class AbstractRequestScanner implements Scanner { return sb.toString(); } - protected String matchRequest(String url, String match) + protected List split(String url, String splitMatch) throws ChessHogScannerException { + RegexResult result = splitRequest(url, splitMatch); + return result.resultList(); + } + + protected Map splitMap(String url, String splitMatch) throws ChessHogScannerException { + RegexResult result = splitRequest(url, splitMatch); + return result.resultMap(); + } + + protected RegexResult splitRequest(String url, String splitMatch) throws ChessHogScannerException { + RegexMatcher matcher = new RegexTypeMatcher( + RegexType.SPLIT, splitMatch + ); + return matchRequest(url, matcher, false); + } + + protected String match(String url, String match) throws ChessHogScannerException { - List result = matchRequest(url, match, true); + List result = match(url, match, true); return result.isEmpty() ? null : result.get(0); } - protected List matchRequest(String url, String match, boolean isFirst) + protected List match(String url, String match, boolean isFirst) throws ChessHogScannerException { + RegexMatcher matcher = new RegexTypeMatcher( + RegexType.FIND, match + ); RegexResult result = matchRequest( url, - new RegexMatcherResult( - match, - isFirst, - false - ) + matcher, + isFirst ); return result.resultList(); } - protected String matchRequest(String url, String startMatch, String endMatch) - throws ChessHogScannerException - { - return matchRequest(url, startMatch, endMatch, true); - } - - protected String matchRequest(String url, String startMatch, String endMatch, boolean isFirst) + protected Map matchMap(String url, String match, boolean isFirst) throws ChessHogScannerException { + RegexMatcher matcher = new RegexTypeMatcher( + RegexType.FIND, match + ); RegexResult result = matchRequest( url, - new RegexMatcherResult( - startMatch, - endMatch, - isFirst, - false - ) + matcher, + isFirst ); - return result.isEmpty() ? null : result.resultList().get(0); + return result.resultMap(); } -/* - protected String matchRequest(String url, String match) + protected String match(String url, String startMatch, String endMatch) throws ChessHogScannerException { - List result = matchRequest(url, match, true); + List result = match(url, startMatch, endMatch, true); return result.isEmpty() ? null : result.get(0); } - protected List matchRequest(String url, String match, boolean isFirst) + protected List match(String url, String startMatch, String endMatch, boolean isFirst) throws ChessHogScannerException { - HttpGet request = new HttpGet(url); - CloseableHttpResponse response = null; - List result = new ArrayList<>(); - try { - response = getClient().execute(request); - Pattern pattern = Pattern.compile(match); - try (BufferedReader br = new BufferedReader( - new InputStreamReader( - response.getEntity().getContent(), - ChessHogScannerConstants.CHARSET - ) - )) { - String line; - while ((line = br.readLine()) != null) { - Matcher matcher = pattern.matcher(line); - if (matcher.find()) { - result.add( - matcher.group(1) - ); - if (isFirst) { - break; - } - } - } - } - } catch (IOException cause) { - throw new ChessHogScannerException( - String.format("Error occurred for requesting URL: %s", url), cause - ); - } finally { - if (response != null) { - try { - response.close(); - } catch (IOException ignored) {} - } - } - return result; + RegexMatcher matcher = new RegexTypeMatcher( + startMatch, endMatch + ); + RegexResult result = matchRequest( + url, + matcher, + isFirst + ); + return result.resultList(); } - protected String matchRequest(String url, String startMatch, String endMatch) + protected Map matchMap(String url, String startMatch, String endMatch, boolean isFirst) throws ChessHogScannerException { - return matchRequest(url, startMatch, endMatch, true); + RegexMatcher matcher = new RegexTypeMatcher( + startMatch, endMatch + ); + RegexResult result = matchRequest( + url, + matcher, + isFirst + ); + return result.resultMap(); } - protected String matchRequest(String url, String startMatch, String endMatch, boolean isFirst) + protected RegexResult matchRequest(String url, RegexMatcher matcher, boolean isFirst) throws ChessHogScannerException { HttpGet request = new HttpGet(url); CloseableHttpResponse response = null; - StringBuilder sb = new StringBuilder(); - try { - response = getClient().execute(request); - Pattern startPattern = Pattern.compile(startMatch); - Pattern endPattern = Pattern.compile(endMatch); - try (BufferedReader br = new BufferedReader( - new InputStreamReader( - response.getEntity().getContent(), - ChessHogScannerConstants.CHARSET - ) - )) { - String line; - boolean isMatch = false; - while ((line = br.readLine()) != null) { - Matcher matcher = isMatch - ? endPattern.matcher(line) - : startPattern.matcher(line); - if (matcher.find()) { - sb.append(line); - if (isMatch && isFirst) { - break; - } - isMatch = !isMatch; - } else { - if (isMatch) { - sb.append(line); - } - } - } - } - } catch (IOException cause) { - throw new ChessHogScannerException( - String.format("Error occurred for requesting URL: %s", url), cause - ); - } finally { - if (response != null) { - try { - response.close(); - } catch (IOException ignored) {} - } - } - return sb.length() > 0 ? sb.toString() : null; - } -*/ - - protected RegexResult matchRequest(String url, RegexMatcher matcher) throws ChessHogScannerException { - HttpGet request = new HttpGet(url); - CloseableHttpResponse response = null; try { response = getClient().execute(request); try (BufferedReader br = new BufferedReader( @@ -240,7 +180,7 @@ public abstract class AbstractRequestScanner implements Scanner { String line; while ((line = br.readLine()) != null) { matcher.match(line); - if (matcher.isBreak()) { + if (isFirst && !matcher.result().isEmpty()) { break; } } @@ -259,23 +199,6 @@ public abstract class AbstractRequestScanner implements Scanner { return matcher.result(); } -/* - protected String urlEncode(String query) throws ChessHogScannerException { - String encodeQuery; - try { - encodeQuery = URLEncoder.encode( - query, ChessHogScannerConstants.CHARSET.name() - ); - } catch (UnsupportedEncodingException cause) { - throw new ChessHogScannerException( - String.format("Unsupported encoding: %s", ChessHogScannerConstants.CHARSET.name()), - cause - ); - } - return encodeQuery; - } -*/ - private CloseableHttpClient getClient() { return HttpClients.createMinimal(); } diff --git a/src/main/java/org/hedgecode/chess/scanner/portal/AbstractSettingsScanner.java b/src/main/java/org/hedgecode/chess/scanner/portal/AbstractSettingsScanner.java index 871c778..58ac5b0 100644 --- a/src/main/java/org/hedgecode/chess/scanner/portal/AbstractSettingsScanner.java +++ b/src/main/java/org/hedgecode/chess/scanner/portal/AbstractSettingsScanner.java @@ -22,9 +22,10 @@ import org.hedgecode.chess.scanner.Settings; import org.hedgecode.chess.scanner.SettingsBuilder; import org.hedgecode.chess.scanner.regex.RegexBuilder; import org.hedgecode.chess.scanner.regex.RegexParams; -import org.hedgecode.chess.scanner.regex.RegexType; import org.hedgecode.chess.scanner.spi.ServiceRegistry; +import static org.hedgecode.chess.scanner.regex.RegexBuilder.Type; + /** * AbstractSettingsScanner * @@ -51,9 +52,12 @@ public abstract class AbstractSettingsScanner extends AbstractRequestScanner imp } protected String assignUrl(String tournamentId, String pageId) { + String tournamentUrl = assignUrlWithParams( + settings.getTournamentUrl(), settings.getGameQueryParams() + ); return RegexBuilder.build( - RegexType.TOURNAMENT, - settings.getTournamentUrl(), + Type.TOURNAMENT, + tournamentUrl, new RegexParams( tournamentId, settings.isTournamentMultiPage() ? pageId : null @@ -66,18 +70,24 @@ public abstract class AbstractSettingsScanner extends AbstractRequestScanner imp ? settings.getGamePgnUrl() : settings.getGameUrl(); return RegexBuilder.build( - RegexType.GAME, - gameUrl, + Type.GAME, + assignUrlWithParams(gameUrl, settings.getGameQueryParams()), new RegexParams(gameId) ); } protected String assignUrl(String query, boolean isUrlEncode) { return RegexBuilder.build( - RegexType.QUERY, + Type.QUERY, settings.getTournamentQuery(), new RegexParams(query, isUrlEncode) ); } + private String assignUrlWithParams(String url, String params) { + return params != null + ? url.concat(params) + : url; + } + } diff --git a/src/main/java/org/hedgecode/chess/scanner/portal/ChessGamesScanner.java b/src/main/java/org/hedgecode/chess/scanner/portal/ChessGamesScanner.java index c07e392..b34e447 100644 --- a/src/main/java/org/hedgecode/chess/scanner/portal/ChessGamesScanner.java +++ b/src/main/java/org/hedgecode/chess/scanner/portal/ChessGamesScanner.java @@ -23,8 +23,6 @@ import java.util.Map; import org.hedgecode.chess.scanner.ChessHogScannerException; import org.hedgecode.chess.scanner.entity.PGNGame; import org.hedgecode.chess.scanner.entity.PGNTournament; -import org.hedgecode.chess.scanner.regex.RegexMatcherResult; -import org.hedgecode.chess.scanner.regex.RegexResult; /** * ChessGamesScanner @@ -49,18 +47,15 @@ public class ChessGamesScanner extends AbstractSettingsScanner { @Override public PGNTournament findTournament(String tournamentName) throws ChessHogScannerException { - RegexResult result = matchRequest( + Map result = matchMap( assignUrl( tournamentName, true ), - new RegexMatcherResult( - getSettings().getTournamentQueryUrlRegex(), - true, - true - ) + getSettings().getTournamentQueryUrlRegex(), + true ); PGNTournament tournament = null; - for (Map.Entry entry : result.resultMap().entrySet()) { + for (Map.Entry entry : result.entrySet()) { if (entry.getValue().contains(tournamentName)) { // todo: contains? tournament = new PGNTournament( entry.getKey(), @@ -91,7 +86,7 @@ public class ChessGamesScanner extends AbstractSettingsScanner { List pageGamesId; int pageId = 0; do { - pageGamesId = matchRequest( + pageGamesId = match( assignUrl( tournament.id(), Integer.toString(++pageId) -- 2.10.0