Series import: extract quantity of the stamps.

Fix #781

Author: php-coder

src/main/java/ru/mystamps/web/controller/SeriesImportController.java

@@ -128,6 +128,9 @@ public String showRequestAndImportSeriesForm(
 			if (parsedData.getIssueYear() != null) {
 				form.setYear(parsedData.getIssueYear());
 			}
+			if (parsedData.getQuantity() != null) {
+				form.setQuantity(parsedData.getQuantity());
+			}
 		}
 
 		model.addAttribute("importSeriesForm", form);

src/main/java/ru/mystamps/web/controller/event/DownloadingSucceededEventListener.java

@@ -90,7 +90,8 @@ public void onApplicationEvent(DownloadingSucceeded event) {
 			info.getCategoryName(),
 			info.getCountryName(),
 			info.getImageUrl(),
-			info.getIssueDate()
+			info.getIssueDate(),
+			info.getQuantity()
 		);
 		importService.saveParsedData(requestId, data);
 	}

src/main/java/ru/mystamps/web/dao/dto/ParsedDataDto.java

@@ -27,4 +27,5 @@ public class ParsedDataDto {
 	private final LinkEntityDto country;
 	private final String imageUrl;
 	private final Integer issueYear;
+	private final Integer quantity;
 }

src/main/java/ru/mystamps/web/dao/dto/SaveParsedDataDbDto.java

@@ -31,14 +31,16 @@ public class SaveParsedDataDbDto {
 	private Integer countryId;
 	private String imageUrl;
 	private Integer releaseYear;
+	private Integer quantity;
 	private Date createdAt;
 	private Date updatedAt;
 
 	public boolean hasAtLeastOneFieldFilled() {
 		return categoryId != null
 			|| countryId != null
 			|| imageUrl != null
-			|| releaseYear != null;
+			|| releaseYear != null
+			|| quantity != null;
 	}
 
 }

src/main/java/ru/mystamps/web/dao/impl/JdbcSeriesImportDao.java

@@ -211,6 +211,7 @@ public void addParsedContent(Integer requestId, SaveParsedDataDbDto data) {
 		params.put("release_year", data.getReleaseYear());
 		params.put("created_at", data.getCreatedAt());
 		params.put("updated_at", data.getUpdatedAt());
+		params.put("quantity", data.getQuantity());
 
 		KeyHolder holder = new GeneratedKeyHolder();
 

src/main/java/ru/mystamps/web/dao/impl/RowMappers.java

@@ -273,8 +273,9 @@ public static ParsedDataDto forParsedDataDto(ResultSet rs, int i) throws SQLExce
 
 		String imageUrl = rs.getString("image_url");
 		Integer releaseYear = JdbcUtils.getInteger(rs, "release_year");
+		Integer quantity = JdbcUtils.getInteger(rs, "quantity");
 
-		return new ParsedDataDto(category, country, imageUrl, releaseYear);
+		return new ParsedDataDto(category, country, imageUrl, releaseYear, quantity);
 	}
 
 	public static ImportRequestInfo forImportRequestInfo(ResultSet rs, int i) throws SQLException {

src/main/java/ru/mystamps/web/service/SeriesImportServiceImpl.java

@@ -185,6 +185,11 @@ public void saveParsedData(Integer requestId, RawParsedDataDto data) {
 			processedData.setReleaseYear(releaseYear);
 		}
 
+		Integer quantity = extractorService.extractQuantity(data.getQuantity());
+		if (quantity != null) {
+			processedData.setQuantity(quantity);
+		}
+		
 		// IMPORTANT: don't add code that modifies database above this line!
 		// @todo #684 Series import: add integration test
 		//  for the case when parsed value don't match database

src/main/java/ru/mystamps/web/service/SeriesInfoExtractorService.java

@@ -23,4 +23,5 @@ public interface SeriesInfoExtractorService {
 	List<Integer> extractCategory(String fragment);
 	List<Integer> extractCountry(String fragment);
 	Integer extractReleaseYear(String fragment);
+	Integer extractQuantity(String fragment);
 }

src/main/java/ru/mystamps/web/service/SeriesInfoExtractorServiceImpl.java

@@ -21,6 +21,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import org.apache.commons.lang3.StringUtils;
@@ -41,6 +42,10 @@ public class SeriesInfoExtractorServiceImpl implements SeriesInfoExtractorServic
 	private static final Pattern RELEASE_YEAR_REGEXP =
 		Pattern.compile("18[4-9][0-9]|19[0-9]{2}|20[0-9]{2}");
 
+	// Regular expression matches number of the stamps in a series (from 1 to 99).
+	private static final Pattern NUMBER_OF_STAMPS_REGEXP =
+		Pattern.compile("([1-9][0-9]?) марок");
+	
 	private final Logger log;
 	private final CategoryService categoryService;
 	private final CountryService countryService;
@@ -144,4 +149,15 @@ public Integer extractReleaseYear(String fragment) {
 		return null;
 	}
 
+	// @todo #781 SeriesInfoExtractorServiceImpl.extractQuantity(): add unit tests
+	// @todo #781 SeriesInfoExtractorServiceImpl.extractQuantity() respect MAX_STAMPS_IN_SERIES
+	@Override
+	public Integer extractQuantity(String fragment) {
+		Matcher matcher = NUMBER_OF_STAMPS_REGEXP.matcher(fragment);
+		if (matcher.find()) {
+			return Integer.valueOf(matcher.group(1));
+		}
+		return null;
+	}
+	
 }

src/main/java/ru/mystamps/web/service/dto/RawParsedDataDto.java

@@ -27,4 +27,5 @@ public class RawParsedDataDto {
 	private final String countryName;
 	private final String imageUrl;
 	private final String releaseYear;
+	private final String quantity;
 }

src/main/java/ru/mystamps/web/util/extractor/SeriesInfo.java

@@ -34,6 +34,7 @@ public class SeriesInfo {
 	private String countryName;
 	private String imageUrl;
 	private String issueDate;
+	private String quantity;
 
 	/**
 	 * Check whether any info about a series is available.
@@ -42,7 +43,8 @@ public boolean isEmpty() {
 		return categoryName == null
 			&& countryName == null
 			&& imageUrl == null
-			&& issueDate == null;
+			&& issueDate == null
+			&& quantity == null;
 	}
 
 }

src/main/java/ru/mystamps/web/util/extractor/SiteParser.java

@@ -136,6 +136,7 @@ public SeriesInfo parse(String htmlPage) {
 		info.setCountryName(extractCountry(body));
 		info.setImageUrl(extractImageUrl(body));
 		info.setIssueDate(extractIssueDate(body));
+		info.setQuantity(extractQuantity(body));
 
 		return info;
 	}
@@ -209,4 +210,20 @@ protected String extractIssueDate(Element body) {
 		return date;
 	}
 
+	protected String extractQuantity(Element body) {
+		String locator = shortDescriptionLocator;
+		if (locator == null) {
+			return null;
+		}
+		
+		Element elem = body.selectFirst(locator);
+		if (elem == null) {
+			return null;
+		}
+		
+		String quantity = elem.text();
+		LOG.debug("Extracted quantity: '{}'", quantity);
+		return quantity;
+	}
+	
 }

src/main/resources/liquibase/version/0.4.xml

@@ -44,5 +44,6 @@
 	<include file="0.4/2017-12-21--solovyov_catalog.xml" relativeToChangelogFile="true" />
 	<include file="0.4/2017-12-21--zagorski_catalog.xml" relativeToChangelogFile="true" />
 	<include file="0.4/2017-12-30--series_import_requests_url_length2.xml" relativeToChangelogFile="true" />
+	<include file="0.4/2018-01-01--add_quantity_to_parsed_data.xml" relativeToChangelogFile="true" />
 
 </databaseChangeLog>

src/main/resources/liquibase/version/0.4/2018-01-01--add_quantity_to_parsed_data.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<databaseChangeLog
+		xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+		xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+		xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
+		http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.1.xsd">
+	
+	<changeSet id="add-quantity-field-to-series_import_parsed_data" author="php-coder" context="scheme">
+		
+		<addColumn tableName="series_import_parsed_data">
+			<column name="quantity" type="INTEGER" afterColumn="image_url" />
+		</addColumn>
+		
+	</changeSet>
+	
+</databaseChangeLog>

src/main/resources/sql/series_import_request_dao_queries.properties

@@ -88,6 +88,7 @@ INSERT \
      , country_id \
      , image_url \
      , release_year \
+     , quantity \
      , created_at \
      , updated_at \
      ) \
@@ -97,6 +98,7 @@ VALUES \
      , :country_id \
      , :image_url \
      , :release_year \
+     , :quantity \
      , :created_at \
      , :updated_at \
      )
@@ -110,6 +112,7 @@ series_import_requests.find_parsed_data_by_request_id = \
         , count.slug AS country_slug \
         , CASE WHEN 'ru' = :lang THEN COALESCE(count.name_ru, count.name) ELSE count.name END AS country_name \
         , pd.release_year \
+        , pd.quantity \
      FROM series_import_parsed_data pd \
 LEFT JOIN categories cat \
        ON cat.id = pd.category_id \

src/test/groovy/ru/mystamps/web/service/SeriesImportServiceImplTest.groovy

@@ -410,7 +410,8 @@ class SeriesImportServiceImplTest extends Specification {
 				Random.categoryName(),
 				Random.countryName(),
 				null, /* imageUrl */
-				Random.issueYear().toString()
+				Random.issueYear().toString(),
+				Random.quantity().toString()
 			)
 		and:
 			extractorService.extractCategory(_ as String) >> Collections.emptyList()
@@ -434,7 +435,8 @@ class SeriesImportServiceImplTest extends Specification {
 				Random.categoryName(),
 				Random.countryName(),
 				expectedImageUrl,
-				Random.issueYear().toString()
+				Random.issueYear().toString(),
+				Random.quantity().toString()
 			)
 		and:
 			extractorService.extractCategory(_ as String) >> Collections.emptyList()
@@ -464,12 +466,14 @@ class SeriesImportServiceImplTest extends Specification {
 			List<Integer> expectedCountryIds = Random.listOfIntegers()
 			Integer expectedCategoryId = expectedCategoryIds.get(0)
 			Integer expectedCountryId = expectedCountryIds.get(0)
+			Integer expectedQuantity = Random.quantity()
 		and:
 			RawParsedDataDto parsedData = new RawParsedDataDto(
 				expectedCategoryName,
 				expectedCountryName,
 				Random.url(),
-				expectedReleaseYear.toString()
+				expectedReleaseYear.toString(),
+				expectedQuantity.toString()
 			)
 		when:
 			service.saveParsedData(expectedRequestId, parsedData)
@@ -479,13 +483,16 @@ class SeriesImportServiceImplTest extends Specification {
 			1 * extractorService.extractCountry(expectedCountryName) >> expectedCountryIds
 		and:
 			1 * extractorService.extractReleaseYear(expectedReleaseYear.toString()) >> expectedReleaseYear
+		and:
+			1 * extractorService.extractQuantity(expectedQuantity.toString()) >> expectedQuantity
 		and:
 			1 * seriesImportDao.addParsedContent(
 				expectedRequestId,
 				{ SaveParsedDataDbDto saveParsedData ->
 					assert saveParsedData?.categoryId  == expectedCategoryId
 					assert saveParsedData?.countryId   == expectedCountryId
 					assert saveParsedData?.releaseYear == expectedReleaseYear
+					assert saveParsedData?.quantity    == expectedQuantity
 					return true
 				}
 			)

src/test/java/ru/mystamps/web/service/TestObjects.java

@@ -202,7 +202,8 @@ public static ParsedDataDto createParsedDataDto() {
 			new LinkEntityDto(Random.id(), categorySlug, categoryName),
 			new LinkEntityDto(Random.id(), countrySlug, countryName),
 			Random.url(),
-			Random.issueYear()
+			Random.issueYear(),
+			Random.quantity()
 		);
 	}
 
@@ -211,7 +212,8 @@ public static RawParsedDataDto createRawParsedDataDto() {
 			Random.categoryName(),
 			Random.countryName(),
 			Random.url(),
-			Random.issueYear().toString()
+			Random.issueYear().toString(),
+			Random.quantity().toString()
 		);
 	}
 

src/test/java/ru/mystamps/web/tests/Random.java

@@ -144,6 +144,13 @@ public static Integer issueYear() {
 		return between(ValidationRules.MIN_RELEASE_YEAR, Year.now().getValue()).integer();
 	}
 
+	public static Integer quantity() {
+		return between(
+			ValidationRules.MIN_STAMPS_IN_SERIES,
+			ValidationRules.MAX_STAMPS_IN_SERIES
+		).integer();
+	}
+	
 	public static String importRequestStatus() {
 		return sample(STATUSES);
 	}

src/test/java/ru/mystamps/web/util/extractor/SiteParserTest.java

@@ -636,6 +636,44 @@ public void extractIssueDateShouldReturnTextOfShortDescriptionLocator() {
 		assertThat(msg, date, equalTo(expectedDate));
 	}
 
+	//
+	// Tests for extractQuantity()
+	//
+	
+	@Test
+	public void extractQuantityShouldReturnNullWhenShortDescriptionLocatorIsNotSet() {
+		parser.setShortDescriptionLocator(null);
+		Element doc = createEmptyDocument();
+		
+		String quantity = parser.extractQuantity(doc);
+		
+		assertThat(quantity, is(nullValue()));
+	}
+	
+	@Test
+	public void extractQuantityShouldReturnNullWhenElementNotFound() {
+		parser.setShortDescriptionLocator(Random.jsoupLocator());
+		Element doc = createEmptyDocument();
+		
+		String quantity = parser.extractQuantity(doc);
+		
+		assertThat(quantity, is(nullValue()));
+	}
+	
+	@Test
+	public void extractQuantityShouldReturnTextOfShortDescriptionLocator() {
+		parser.setShortDescriptionLocator("#desc");
+		
+		String expectedQuantity = Random.quantity().toString();
+		String html = String.format("<div id='desc'>%s</div>", expectedQuantity);
+		Element doc = createDocumentFromText(html);
+		
+		String quantity = parser.extractQuantity(doc);
+		
+		String msg = String.format("couldn't extract quantity from '%s'", doc);
+		assertThat(msg, quantity, equalTo(expectedQuantity));
+	}
+	
 	private static String describe(SiteParser parser) {
 		StringBuilder sb = new StringBuilder();
 		sb.append("SiteParser[name=")