Import series: add support for extracting issue year.

Addressed to #693

Author: php-coder

src/main/java/ru/mystamps/web/controller/SeriesController.java

@@ -131,6 +131,7 @@ public void showForm(
 		@Category @RequestParam(name = "category", required = false) LinkEntityDto category,
 		@Country @RequestParam(name = "country", required = false) LinkEntityDto country,
 		@RequestParam(name = "image_url", required = false) String imageUrl,
+		@RequestParam(name = "year", required = false) Integer year,
 		Model model,
 		Locale userLocale) {
 
@@ -161,6 +162,10 @@ public void showForm(
 			addSeriesForm.setImageUrl(imageUrl);
 		}
 
+		if (year != null) {
+			addSeriesForm.setYear(year);
+		}
+		
 		model.addAttribute("addSeriesForm", addSeriesForm);
 	}
 

src/main/java/ru/mystamps/web/controller/event/DownloadingSucceededEventListener.java

@@ -89,7 +89,8 @@ public void onApplicationEvent(DownloadingSucceeded event) {
 		RawParsedDataDto data = new RawParsedDataDto(
 			info.getCategoryName(),
 			info.getCountryName(),
-			info.getImageUrl()
+			info.getImageUrl(),
+			info.getIssueDate()
 		);
 		importService.saveParsedData(requestId, data);
 	}

src/main/java/ru/mystamps/web/dao/dto/ParsedDataDto.java

@@ -26,4 +26,5 @@ public class ParsedDataDto {
 	private final EntityWithSlugDto category;
 	private final EntityWithSlugDto country;
 	private final String imageUrl;
+	private final Integer issueYear;
 }

src/main/java/ru/mystamps/web/dao/dto/SaveParsedDataDbDto.java

@@ -30,13 +30,15 @@ public class SaveParsedDataDbDto {
 	private Integer categoryId;
 	private Integer countryId;
 	private String imageUrl;
+	private Integer releaseYear;
 	private Date createdAt;
 	private Date updatedAt;
 
 	public boolean hasAtLeastOneFieldFilled() {
 		return categoryId != null
 			|| countryId != null
-			|| imageUrl != null;
+			|| imageUrl != null
+			|| releaseYear != null;
 	}
 
 }

src/main/java/ru/mystamps/web/dao/impl/JdbcSeriesImportDao.java

@@ -170,6 +170,7 @@ public void addParsedContent(Integer requestId, SaveParsedDataDbDto data) {
 		params.put("category_id", data.getCategoryId());
 		params.put("country_id", data.getCountryId());
 		params.put("image_url", data.getImageUrl());
+		params.put("release_year", data.getReleaseYear());
 		params.put("created_at", data.getCreatedAt());
 		params.put("updated_at", data.getUpdatedAt());
 

src/main/java/ru/mystamps/web/dao/impl/RowMappers.java

@@ -288,8 +288,9 @@ public static ParsedDataDto forParsedDataDto(ResultSet rs, int i) throws SQLExce
 		}
 
 		String imageUrl = rs.getString("image_url");
+		Integer releaseYear = JdbcUtils.getInteger(rs, "release_year");
 
-		return new ParsedDataDto(category, country, imageUrl);
+		return new ParsedDataDto(category, country, imageUrl, releaseYear);
 	}
 
 }

src/main/java/ru/mystamps/web/service/SeriesImportServiceImpl.java

@@ -147,6 +147,11 @@ public void saveParsedData(Integer requestId, RawParsedDataDto data) {
 			processedData.setCountryId(countryIds.get(0));
 		}
 
+		Integer releaseYear = extractorService.extractReleaseYear(data.getReleaseYear());
+		if (releaseYear != null) {
+			processedData.setReleaseYear(releaseYear);
+		}
+		
 		// TODO: handle it gracefully by publishing ParsingFailed event
 		Validate.validState(processedData.hasAtLeastOneFieldFilled(), "");
 

src/main/java/ru/mystamps/web/service/SeriesInfoExtractorService.java

@@ -22,4 +22,5 @@
 public interface SeriesInfoExtractorService {
 	List<Integer> extractCategory(String fragment);
 	List<Integer> extractCountry(String fragment);
+	Integer extractReleaseYear(String fragment);
 }

src/main/java/ru/mystamps/web/service/SeriesInfoExtractorServiceImpl.java

@@ -21,6 +21,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.regex.Pattern;
 
 import org.apache.commons.lang3.StringUtils;
 
@@ -33,6 +34,9 @@
 @RequiredArgsConstructor
 public class SeriesInfoExtractorServiceImpl implements SeriesInfoExtractorService {
 
+	// Regular expression matches release year of the stamps (from 1840 till 2099).
+	private static final Pattern RELEASE_YEAR_REGEXP = Pattern.compile("18[4-9][0-9]|19[0-9]{2}|20[0-9]{2}");
+	
 	private final Logger log;
 	private final CategoryService categoryService;
 	private final CountryService countryService;
@@ -107,4 +111,33 @@ public List<Integer> extractCountry(String fragment) {
 		return Collections.emptyList();
 	}
 
+	@Override
+	public Integer extractReleaseYear(String fragment) {
+		if (StringUtils.isBlank(fragment)) {
+			return null;
+		}
+		
+		log.debug("Determining release year from a fragment: '{}'", fragment);
+		
+		String[] candidates = StringUtils.split(fragment);
+		for (String candidate : candidates) {
+			if (!RELEASE_YEAR_REGEXP.matcher(candidate).matches()) {
+				continue;
+			}
+			
+			try {
+				Integer year = Integer.valueOf(candidate);
+				log.debug("Release year is {}", year);
+				return year;
+				
+			} catch (NumberFormatException ignored) {
+				// continue with the next element
+			}
+		}
+		
+		log.debug("Could not extract release year from a fragment");
+		
+		return null;
+	}
+	
 }

src/main/java/ru/mystamps/web/service/dto/RawParsedDataDto.java

@@ -26,4 +26,5 @@ public class RawParsedDataDto {
 	private final String categoryName;
 	private final String countryName;
 	private final String imageUrl;
+	private final String releaseYear;
 }

src/main/java/ru/mystamps/web/util/extractor/SeriesInfo.java

@@ -29,14 +29,16 @@ public class SeriesInfo {
 	private String categoryName;
 	private String countryName;
 	private String imageUrl;
+	private String issueDate;
 
 	/**
 	 * Check whether any info about a series is available.
 	 */
 	public boolean isEmpty() {
 		return categoryName == null
 			&& countryName == null
-			&& imageUrl == null;
+			&& imageUrl == null
+			&& issueDate == null;
 	}
 
 }

src/main/java/ru/mystamps/web/util/extractor/SiteParser.java

@@ -41,6 +41,7 @@ public class SiteParser {
 	private String countryLocator;
 	private String shortDescriptionLocator;
 	private String imageUrlLocator;
+	private String issueDateLocator;
 
 	public boolean setField(String name, String value) {
 		Validate.validState(StringUtils.isNotBlank(name), "Field name must be non-null");
@@ -80,6 +81,11 @@ public boolean setField(String name, String value) {
 				valid = true;
 				break;
 
+			case "issue-date-locator":
+				setIssueDateLocator(value);
+				valid = true;
+				break;
+			
 			default:
 				break;
 		}
@@ -95,6 +101,7 @@ public boolean isFullyInitialized() {
 				|| countryLocator != null
 				|| shortDescriptionLocator != null
 				|| imageUrlLocator != null
+				|| issueDateLocator != null
 			);
 	}
 
@@ -122,6 +129,7 @@ public SeriesInfo parse(String htmlPage) {
 		info.setCategoryName(extractCategory(body));
 		info.setCountryName(extractCountry(body));
 		info.setImageUrl(extractImageUrl(body));
+		info.setIssueDate(extractIssueDate(body));
 
 		return info;
 	}
@@ -178,4 +186,20 @@ private String extractImageUrl(Element body) {
 		return StringUtils.trimToNull(url);
 	}
 
+	private String extractIssueDate(Element body) {
+		String locator = ObjectUtils.firstNonNull(issueDateLocator, shortDescriptionLocator);
+		if (locator == null) {
+			return null;
+		}
+		
+		Elements elements = body.select(locator);
+		if (elements.isEmpty()) {
+			return null;
+		}
+		
+		String date = elements.first().text();
+		LOG.debug("Extracted issue date: '{}'", date);
+		return date;
+	}
+	
 }

src/main/resources/application-test.properties

@@ -48,6 +48,7 @@ app.site-parser[0].matched-url = http://127.0.0.1:8080
 app.site-parser[0].category-locator = #category_name
 app.site-parser[0].country-locator = #country_name
 app.site-parser[0].image-url-locator = #series-image-link-1
+app.site-parser[0].issue-date-locator = #issue_date
 
 app.site-parser[1].name = localhost-by-name
 app.site-parser[1].matched-url = http://localhost:8080

src/main/resources/application-travis.properties

@@ -45,6 +45,7 @@ app.site-parser[0].matched-url = http://127.0.0.1:8080
 app.site-parser[0].category-locator = #category_name
 app.site-parser[0].country-locator = #country_name
 app.site-parser[0].image-url-locator = #series-image-link-1
+app.site-parser[0].issue-date-locator = #issue_date
 
 app.site-parser[1].name = localhost-by-name
 app.site-parser[1].matched-url = http://localhost:8080

src/main/resources/liquibase/version/0.4.xml

@@ -34,5 +34,6 @@
 	<include file="0.4/2017-10-31--non_nullable_catalog_codes.xml" relativeToChangelogFile="true" />
 	<include file="0.4/2017-11-08--import_series.xml" relativeToChangelogFile="true" />
 	<include file="0.4/2017-11-09--series_import_requests_url_length.xml" relativeToChangelogFile="true" />
+	<include file="0.4/2017-11-09--series_import_parsed_data_release_year_field.xml" relativeToChangelogFile="true" />
 
 </databaseChangeLog>

src/main/resources/liquibase/version/0.4/2017-11-08--import_series.xml

@@ -139,6 +139,7 @@
 			<column name="perforated" valueBoolean="false" />
 			<column name="category_id" valueNumeric="1" />
 			<column name="country_id" valueNumeric="1" />
+			<column name="release_year" valueNumeric="2000" />
 			<column name="created_at" valueComputed="${NOW}" />
 			<column name="created_by" valueComputed="(SELECT id FROM users WHERE role = 'USER' ORDER by id LIMIT 1)" />
 			<column name="updated_at" valueComputed="${NOW}" />

src/main/resources/liquibase/version/0.4/2017-11-09--series_import_parsed_data_release_year_field.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<databaseChangeLog
+		xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+		xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+		xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
+		http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.0.xsd">
+	
+	<changeSet id="add-release_year-column-to-series_import_parsed_data-table" author="php-coder" context="scheme">
+		
+		<addColumn tableName="series_import_parsed_data">
+			<column name="release_year" type="INTEGER" />
+		</addColumn>
+		
+	</changeSet>
+	
+</databaseChangeLog>

src/main/resources/sql/series_import_request_dao_queries.properties

@@ -67,6 +67,7 @@ INSERT \
      , category_id \
      , country_id \
      , image_url \
+     , release_year \
      , created_at \
      , updated_at \
      ) \
@@ -75,6 +76,7 @@ VALUES \
      , :category_id \
      , :country_id \
      , :image_url \
+     , :release_year \
      , :created_at \
      , :updated_at \
      )
@@ -85,6 +87,7 @@ series_import_requests.find_parsed_data_by_request_id = \
         , CASE WHEN 'ru' = :lang THEN COALESCE(cat.name_ru, cat.name) ELSE cat.name END AS category_name \
         , count.slug AS country_slug \
         , CASE WHEN 'ru' = :lang THEN COALESCE(count.name_ru, count.name) ELSE count.name END AS country_name \
+        , pd.release_year \
      FROM series_import_parsed_data pd \
 LEFT JOIN categories cat \
        ON cat.id = pd.category_id \

src/main/webapp/WEB-INF/views/series/import/info.html

@@ -99,7 +99,7 @@ <h3 th:text="#{t_gathered_data}">
 					<form id="start-import-series-form" method="get" action="../add.html" th:action="@{${ADD_SERIES_PAGE}}">
 						<div class="table-responsive">
 							<table class="table"
-								th:with="hasCategory=${parsedData.category != null},hasCountry=${parsedData.country != null},hasImageUrl=${parsedData.imageUrl != null}">
+								th:with="hasCategory=${parsedData.category != null},hasCountry=${parsedData.country != null},hasImageUrl=${parsedData.imageUrl != null},hasIssueYear=${parsedData.issueYear != null}">
 
 								<tr th:if="${hasCategory}">
 									<th th:text="#{t_category}">Category</th>
@@ -118,7 +118,12 @@ <h3 th:text="#{t_gathered_data}">
 									<td id="parsed-image-url" th:text="${parsedData.imageUrl}">
 										http://localhost:8080/image/1
 									</td>
-
+								</tr>
+								<tr th:if="${hasIssueYear}">
+									<th th:text="#{t_year}">Year</th>
+									<td id="parsed-issue-year" th:text="${parsedData.issueYear}">
+										1960
+									</td>
 								</tr>
 								<tr>
 									<td></td>
@@ -141,6 +146,12 @@ <h3 th:text="#{t_gathered_data}">
 											value="http://localhost:8080/image/1"
 											th:if="${hasImageUrl}"
 											th:value="${parsedData.imageUrl}" />
+										<input
+											type="hidden"
+											name="year"
+											value="1960"
+											th:if="${hasIssueYear}"
+											th:value="${parsedData.issueYear}" />
 										<input
 											type="submit"
 											class="btn btn-primary"

src/test/java/ru/mystamps/web/service/TestObjects.java

@@ -196,7 +196,8 @@ public static ParsedDataDto createParsedDataDto() {
 		return new ParsedDataDto(
 			new EntityWithSlugDto(categoryName, categorySlug),
 			new EntityWithSlugDto(countryName, countrySlug),
-			Random.url()
+			Random.url(),
+			Random.issueYear()
 		);
 	}
 

src/test/java/ru/mystamps/web/tests/Random.java

@@ -17,6 +17,7 @@
  */
 package ru.mystamps.web.tests;
 
+import java.time.Year;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -84,6 +85,10 @@ public static String countryName() {
 			.english();
 	}
 
+	public static Integer issueYear() {
+		return between(ValidationRules.MIN_RELEASE_YEAR, Year.now().getValue()).integer();
+	}
+	
 	public static String importRequestStatus() {
 		return sample(STATUSES);
 	}