Series import: extract info about perforation.

Fix #782

Author: php-coder

src/main/java/ru/mystamps/web/controller/SeriesImportController.java

@@ -87,6 +87,7 @@ public String processRequestImportForm(
 		return redirectTo(Url.REQUEST_IMPORT_PAGE, requestId);
 	}
 
+	@SuppressWarnings({ "PMD.ModifiedCyclomaticComplexity", "PMD.NPathComplexity" })
 	@GetMapping(Url.REQUEST_IMPORT_PAGE)
 	public String showRequestAndImportSeriesForm(
 		@PathVariable("id") Integer requestId,
@@ -131,6 +132,9 @@ public String showRequestAndImportSeriesForm(
 			if (parsedData.getQuantity() != null) {
 				form.setQuantity(parsedData.getQuantity());
 			}
+			if (parsedData.getPerforated() != null) {
+				form.setPerforated(parsedData.getPerforated());
+			}
 		}
 
 		model.addAttribute("importSeriesForm", form);

src/main/java/ru/mystamps/web/controller/event/DownloadingSucceededEventListener.java

@@ -91,7 +91,8 @@ public void onApplicationEvent(DownloadingSucceeded event) {
 			info.getCountryName(),
 			info.getImageUrl(),
 			info.getIssueDate(),
-			info.getQuantity()
+			info.getQuantity(),
+			info.getPerforated()
 		);
 		importService.saveParsedData(requestId, data);
 	}

src/main/java/ru/mystamps/web/dao/dto/ParsedDataDto.java

@@ -28,4 +28,5 @@ public class ParsedDataDto {
 	private final String imageUrl;
 	private final Integer issueYear;
 	private final Integer quantity;
+	private final Boolean perforated;
 }

src/main/java/ru/mystamps/web/dao/dto/SaveParsedDataDbDto.java

@@ -32,6 +32,7 @@ public class SaveParsedDataDbDto {
 	private String imageUrl;
 	private Integer releaseYear;
 	private Integer quantity;
+	private Boolean perforated;
 	private Date createdAt;
 	private Date updatedAt;
 
@@ -40,7 +41,8 @@ public boolean hasAtLeastOneFieldFilled() {
 			|| countryId != null
 			|| imageUrl != null
 			|| releaseYear != null
-			|| quantity != null;
+			|| quantity != null
+			|| perforated != null;
 	}
 
 }

src/main/java/ru/mystamps/web/dao/impl/JdbcSeriesImportDao.java

@@ -212,6 +212,7 @@ public void addParsedContent(Integer requestId, SaveParsedDataDbDto data) {
 		params.put("created_at", data.getCreatedAt());
 		params.put("updated_at", data.getUpdatedAt());
 		params.put("quantity", data.getQuantity());
+		params.put("perforated", data.getPerforated());
 
 		KeyHolder holder = new GeneratedKeyHolder();
 

src/main/java/ru/mystamps/web/dao/impl/JdbcUtils.java

@@ -38,6 +38,16 @@ public static Integer getInteger(ResultSet resultSet, String fieldName) throws S
 		return Integer.valueOf(value);
 	}
 
+	@SuppressWarnings("PMD.PrematureDeclaration")
+	public static Boolean getBoolean(ResultSet resultSet, String fieldName) throws SQLException {
+		boolean value = resultSet.getBoolean(fieldName);
+		if (resultSet.wasNull()) {
+			return null;
+		}
+		
+		return Boolean.valueOf(value);
+	}
+	
 	/**
 	 * @author Sergey Chechenev
 	 */

src/main/java/ru/mystamps/web/dao/impl/RowMappers.java

@@ -274,8 +274,9 @@ public static ParsedDataDto forParsedDataDto(ResultSet rs, int i) throws SQLExce
 		String imageUrl = rs.getString("image_url");
 		Integer releaseYear = JdbcUtils.getInteger(rs, "release_year");
 		Integer quantity = JdbcUtils.getInteger(rs, "quantity");
+		Boolean perforated = JdbcUtils.getBoolean(rs, "perforated");
 
-		return new ParsedDataDto(category, country, imageUrl, releaseYear, quantity);
+		return new ParsedDataDto(category, country, imageUrl, releaseYear, quantity, perforated);
 	}
 
 	public static ImportRequestInfo forImportRequestInfo(ResultSet rs, int i) throws SQLException {

src/main/java/ru/mystamps/web/service/SeriesImportServiceImpl.java

@@ -190,6 +190,11 @@ public void saveParsedData(Integer requestId, RawParsedDataDto data) {
 			processedData.setQuantity(quantity);
 		}
 
+		Boolean perforated = extractorService.extractPerforated(data.getPerforated());
+		if (perforated != null) {
+			processedData.setPerforated(perforated);
+		}
+		
 		// IMPORTANT: don't add code that modifies database above this line!
 		// @todo #684 Series import: add integration test
 		//  for the case when parsed value don't match database

src/main/java/ru/mystamps/web/service/SeriesInfoExtractorService.java

@@ -24,4 +24,5 @@ public interface SeriesInfoExtractorService {
 	List<Integer> extractCountry(String fragment);
 	Integer extractReleaseYear(String fragment);
 	Integer extractQuantity(String fragment);
+	Boolean extractPerforated(String fragment);
 }

src/main/java/ru/mystamps/web/service/SeriesInfoExtractorServiceImpl.java

@@ -164,4 +164,25 @@ public Integer extractQuantity(String fragment) {
 		return null;
 	}
 
+	// @todo #782 SeriesInfoExtractorServiceImpl.extractPerforated(): add unit tests
+	// @todo #782 Series import: add integration test for extracting perforation flag
+	@Override
+	public Boolean extractPerforated(String fragment) {
+		if (StringUtils.isBlank(fragment)) {
+			return null;
+		}
+		
+		log.debug("Determining perforation from a fragment: '{}'", fragment);
+		
+		boolean withoutPerforation = StringUtils.contains(fragment, "б/з");
+		if (withoutPerforation) {
+			log.debug("Perforation is false");
+			return Boolean.FALSE;
+		}
+		
+		log.debug("Could not extract perforation info from a fragment");
+		
+		return null;
+	}
+	
 }

src/main/java/ru/mystamps/web/service/dto/RawParsedDataDto.java

@@ -28,4 +28,5 @@ public class RawParsedDataDto {
 	private final String imageUrl;
 	private final String releaseYear;
 	private final String quantity;
+	private final String perforated;
 }

src/main/java/ru/mystamps/web/util/extractor/SeriesInfo.java

@@ -35,6 +35,7 @@ public class SeriesInfo {
 	private String imageUrl;
 	private String issueDate;
 	private String quantity;
+	private String perforated;
 
 	/**
 	 * Check whether any info about a series is available.
@@ -44,7 +45,8 @@ public boolean isEmpty() {
 			&& countryName == null
 			&& imageUrl == null
 			&& issueDate == null
-			&& quantity == null;
+			&& quantity == null
+			&& perforated == null;
 	}
 
 }

src/main/java/ru/mystamps/web/util/extractor/SiteParser.java

@@ -137,6 +137,7 @@ public SeriesInfo parse(String htmlPage) {
 		info.setImageUrl(extractImageUrl(body));
 		info.setIssueDate(extractIssueDate(body));
 		info.setQuantity(extractQuantity(body));
+		info.setPerforated(extractPerforated(body));
 
 		return info;
 	}
@@ -226,4 +227,21 @@ protected String extractQuantity(Element body) {
 		return quantity;
 	}
 
+	// @todo #782 SiteParser: reduce duplication between extractQuantity() and extractPerforated()
+	protected String extractPerforated(Element body) {
+		String locator = shortDescriptionLocator;
+		if (locator == null) {
+			return null;
+		}
+		
+		Element elem = body.selectFirst(locator);
+		if (elem == null) {
+			return null;
+		}
+		
+		String perforated = elem.text();
+		LOG.debug("Extracted perforated flag: '{}'", perforated);
+		return perforated;
+	}
+	
 }

src/main/resources/liquibase/version/0.4.xml

@@ -45,5 +45,6 @@
 	<include file="0.4/2017-12-21--zagorski_catalog.xml" relativeToChangelogFile="true" />
 	<include file="0.4/2017-12-30--series_import_requests_url_length2.xml" relativeToChangelogFile="true" />
 	<include file="0.4/2018-01-01--add_quantity_to_parsed_data.xml" relativeToChangelogFile="true" />
+	<include file="0.4/2018-01-01--add_perforated_to_parsed_data.xml" relativeToChangelogFile="true" />
 
 </databaseChangeLog>

src/main/resources/liquibase/version/0.4/2018-01-01--add_perforated_to_parsed_data.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<databaseChangeLog
+		xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+		xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+		xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
+		http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.1.xsd">
+	
+	<changeSet id="add-perforated-field-to-series_import_parsed_data" author="php-coder" context="scheme">
+		
+		<addColumn tableName="series_import_parsed_data">
+			<column name="perforated" type="BOOLEAN" afterColumn="image_url" />
+		</addColumn>
+		
+	</changeSet>
+	
+</databaseChangeLog>

src/main/resources/sql/series_import_request_dao_queries.properties

@@ -89,6 +89,7 @@ INSERT \
      , image_url \
      , release_year \
      , quantity \
+     , perforated \
      , created_at \
      , updated_at \
      ) \
@@ -99,6 +100,7 @@ VALUES \
      , :image_url \
      , :release_year \
      , :quantity \
+     , :perforated \
      , :created_at \
      , :updated_at \
      )
@@ -113,6 +115,7 @@ series_import_requests.find_parsed_data_by_request_id = \
         , CASE WHEN 'ru' = :lang THEN COALESCE(count.name_ru, count.name) ELSE count.name END AS country_name \
         , pd.release_year \
         , pd.quantity \
+        , pd.perforated \
      FROM series_import_parsed_data pd \
 LEFT JOIN categories cat \
        ON cat.id = pd.category_id \

src/test/groovy/ru/mystamps/web/service/SeriesImportServiceImplTest.groovy

@@ -411,7 +411,8 @@ class SeriesImportServiceImplTest extends Specification {
 				Random.countryName(),
 				null, /* imageUrl */
 				Random.issueYear().toString(),
-				Random.quantity().toString()
+				Random.quantity().toString(),
+				String.valueOf(Random.perforated())
 			)
 		and:
 			extractorService.extractCategory(_ as String) >> Collections.emptyList()
@@ -436,7 +437,8 @@ class SeriesImportServiceImplTest extends Specification {
 				Random.countryName(),
 				expectedImageUrl,
 				Random.issueYear().toString(),
-				Random.quantity().toString()
+				Random.quantity().toString(),
+				String.valueOf(Random.perforated())
 			)
 		and:
 			extractorService.extractCategory(_ as String) >> Collections.emptyList()
@@ -467,13 +469,15 @@ class SeriesImportServiceImplTest extends Specification {
 			Integer expectedCategoryId = expectedCategoryIds.get(0)
 			Integer expectedCountryId = expectedCountryIds.get(0)
 			Integer expectedQuantity = Random.quantity()
+			Boolean expectedPerforated = Random.perforated()
 		and:
 			RawParsedDataDto parsedData = new RawParsedDataDto(
 				expectedCategoryName,
 				expectedCountryName,
 				Random.url(),
 				expectedReleaseYear.toString(),
-				expectedQuantity.toString()
+				expectedQuantity.toString(),
+				expectedPerforated.toString()
 			)
 		when:
 			service.saveParsedData(expectedRequestId, parsedData)
@@ -485,6 +489,8 @@ class SeriesImportServiceImplTest extends Specification {
 			1 * extractorService.extractReleaseYear(expectedReleaseYear.toString()) >> expectedReleaseYear
 		and:
 			1 * extractorService.extractQuantity(expectedQuantity.toString()) >> expectedQuantity
+		and:
+			1 * extractorService.extractPerforated(expectedPerforated.toString()) >> expectedPerforated
 		and:
 			1 * seriesImportDao.addParsedContent(
 				expectedRequestId,
@@ -493,6 +499,7 @@ class SeriesImportServiceImplTest extends Specification {
 					assert saveParsedData?.countryId   == expectedCountryId
 					assert saveParsedData?.releaseYear == expectedReleaseYear
 					assert saveParsedData?.quantity    == expectedQuantity
+					assert saveParsedData?.perforated  == expectedPerforated
 					return true
 				}
 			)

src/test/java/ru/mystamps/web/service/TestObjects.java

@@ -203,7 +203,8 @@ public static ParsedDataDto createParsedDataDto() {
 			new LinkEntityDto(Random.id(), countrySlug, countryName),
 			Random.url(),
 			Random.issueYear(),
-			Random.quantity()
+			Random.quantity(),
+			Random.perforated()
 		);
 	}
 
@@ -213,7 +214,8 @@ public static RawParsedDataDto createRawParsedDataDto() {
 			Random.countryName(),
 			Random.url(),
 			Random.issueYear().toString(),
-			Random.quantity().toString()
+			Random.quantity().toString(),
+			String.valueOf(Random.perforated())
 		);
 	}
 

src/test/java/ru/mystamps/web/tests/Random.java

@@ -39,6 +39,7 @@
 import ru.mystamps.web.service.TestObjects;
 import ru.mystamps.web.validation.ValidationRules;
 
+import static io.qala.datagen.RandomShortApi.bool;
 import static io.qala.datagen.RandomShortApi.integer;
 import static io.qala.datagen.RandomShortApi.sample;
 import static io.qala.datagen.RandomShortApi.sampleMultiple;
@@ -151,6 +152,10 @@ public static Integer quantity() {
 		).integer();
 	}
 
+	public static boolean perforated() {
+		return bool();
+	}
+	
 	public static String importRequestStatus() {
 		return sample(STATUSES);
 	}

src/test/java/ru/mystamps/web/util/extractor/SiteParserTest.java

@@ -674,6 +674,44 @@ public void extractQuantityShouldReturnTextOfShortDescriptionLocator() {
 		assertThat(msg, quantity, equalTo(expectedQuantity));
 	}
 
+	//
+	// Tests for extractPerforated()
+	//
+	
+	@Test
+	public void extractPerforatedShouldReturnNullWhenShortDescriptionLocatorIsNotSet() {
+		parser.setShortDescriptionLocator(null);
+		Element doc = createEmptyDocument();
+		
+		String perforated = parser.extractPerforated(doc);
+		
+		assertThat(perforated, is(nullValue()));
+	}
+	
+	@Test
+	public void extractPerforatedShouldReturnNullWhenElementNotFound() {
+		parser.setShortDescriptionLocator(Random.jsoupLocator());
+		Element doc = createEmptyDocument();
+		
+		String perforated = parser.extractPerforated(doc);
+		
+		assertThat(perforated, is(nullValue()));
+	}
+	
+	@Test
+	public void extractPerforatedShouldReturnTextOfShortDescriptionLocator() {
+		parser.setShortDescriptionLocator("#desc");
+		
+		String expectedValue = String.valueOf(Random.perforated());
+		String html = String.format("<div id='desc'>%s</div>", expectedValue);
+		Element doc = createDocumentFromText(html);
+		
+		String perforated = parser.extractPerforated(doc);
+		
+		String msg = String.format("couldn't extract perforated flag from '%s'", doc);
+		assertThat(msg, perforated, equalTo(expectedValue));
+	}
+	
 	private static String describe(SiteParser parser) {
 		StringBuilder sb = new StringBuilder();
 		sb.append("SiteParser[name=")