Skip to content

Commit 29f6dc9

Browse files
committed
/series/import/request: implement ability to import series from external site.
Fix #660
1 parent 15d2ba3 commit 29f6dc9

File tree

78 files changed

+3367
-9
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

78 files changed

+3367
-9
lines changed

NEWS.txt

+1
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
- (functionality) preview now is generated after uploading an image
1717
- (functionality) add interface for adding buyers and sellers
1818
- (functionality) add capability to specify image URL (as alternative to providing a file)
19+
- (functionality) admin can import a series from an external site
1920

2021
0.3
2122
- (functionality) implemented possibility to user to add series to his collection

pom.xml

+8
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,12 @@
4040
<version>${hibernate-validator.version}</version>
4141
</dependency>
4242

43+
<dependency>
44+
<groupId>org.jsoup</groupId>
45+
<artifactId>jsoup</artifactId>
46+
<version>${jsoup.version}</version>
47+
</dependency>
48+
4349
<dependency>
4450
<groupId>org.slf4j</groupId>
4551
<artifactId>slf4j-api</artifactId>
@@ -522,6 +528,8 @@
522528
<!-- Don't forget to update version in the Url class -->
523529
<jquery.version>1.9.1</jquery.version>
524530

531+
<jsoup.version>1.10.3</jsoup.version>
532+
525533
<!-- Redefine default value from spring-boot-dependencies (https://github.com/spring-projects/spring-boot/blob/v1.5.7.RELEASE/spring-boot-dependencies/pom.xml) -->
526534
<junit.version>4.12</junit.version>
527535

src/main/java/ru/mystamps/web/Db.java

+4
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,10 @@ public static final class Series {
4040
public static final int COMMENT_LENGTH = 255;
4141
}
4242

43+
public static final class SeriesImportRequest {
44+
public static final int URL_LENGTH = 255;
45+
}
46+
4347
public static final class SuspiciousActivity {
4448
public static final int PAGE_URL_LENGTH = 100;
4549
public static final int METHOD_LENGTH = 7;

src/main/java/ru/mystamps/web/Url.java

+4
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,9 @@ public final class Url {
5353
public static final String ADD_IMAGE_SERIES_PAGE = "/series/{id}/image";
5454
public static final String SEARCH_SERIES_BY_CATALOG = "/series/search/by_catalog";
5555

56+
public static final String REQUEST_IMPORT_SERIES_PAGE = "/series/import/request";
57+
public static final String REQUEST_IMPORT_PAGE = "/series/import/request/{id}";
58+
5659
public static final String SUGGEST_SERIES_COUNTRY = "/suggest/series_country";
5760

5861
public static final String ADD_CATEGORY_PAGE = "/category/add";
@@ -135,6 +138,7 @@ public static Map<String, String> asMap(boolean serveContentFromSingleHost) {
135138
map.put("INFO_SERIES_PAGE", INFO_SERIES_PAGE);
136139
map.put("ADD_IMAGE_SERIES_PAGE", ADD_IMAGE_SERIES_PAGE);
137140
map.put("SEARCH_SERIES_BY_CATALOG", SEARCH_SERIES_BY_CATALOG);
141+
map.put("REQUEST_IMPORT_SERIES_PAGE", REQUEST_IMPORT_SERIES_PAGE);
138142
map.put("SUGGEST_SERIES_COUNTRY", SUGGEST_SERIES_COUNTRY);
139143
map.put("ADD_CATEGORY_PAGE", ADD_CATEGORY_PAGE);
140144
map.put("INFO_CATEGORY_PAGE", INFO_CATEGORY_PAGE);

src/main/java/ru/mystamps/web/config/ApplicationContext.java

+1
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ public static PropertySourcesPlaceholderConfigurer getPropertySourcesPlaceholder
5151
new ClassPathResource("sql/user_dao_queries.properties"),
5252
new ClassPathResource("sql/users_activation_dao_queries.properties"),
5353
new ClassPathResource("sql/series_dao_queries.properties"),
54+
new ClassPathResource("sql/series_import_request_dao_queries.properties"),
5455
new ClassPathResource("sql/series_sales_dao_queries.properties"),
5556
new ClassPathResource("sql/suspicious_activity_dao_queries.properties"),
5657
new ClassPathResource("sql/transaction_participants_dao_queries.properties")

src/main/java/ru/mystamps/web/config/ControllersConfig.java

+10
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
*/
1818
package ru.mystamps.web.config;
1919

20+
import org.springframework.context.ApplicationEventPublisher;
2021
import org.springframework.context.MessageSource;
2122
import org.springframework.context.annotation.Bean;
2223
import org.springframework.context.annotation.Configuration;
@@ -33,6 +34,7 @@ public class ControllersConfig {
3334

3435
private final ServicesConfig servicesConfig;
3536
private final MessageSource messageSource;
37+
private final ApplicationEventPublisher eventPublisher;
3638

3739
@Bean
3840
public AccountController getAccountController() {
@@ -109,6 +111,14 @@ public SeriesController getSeriesController() {
109111
);
110112
}
111113

114+
@Bean
115+
public SeriesImportController getSeriesImportController() {
116+
return new SeriesImportController(
117+
servicesConfig.getSeriesImportService(),
118+
eventPublisher
119+
);
120+
}
121+
112122
@Bean
113123
public SiteController getSiteController() {
114124
return new SiteController(

src/main/java/ru/mystamps/web/config/DaoConfig.java

+5
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,11 @@ public SeriesDao getSeriesDao() {
9797
return new JdbcSeriesDao(jdbcTemplate);
9898
}
9999

100+
@Bean
101+
public SeriesImportDao getSeriesImportDao() {
102+
return new JdbcSeriesImportDao(jdbcTemplate);
103+
}
104+
100105
@Bean
101106
public SeriesSalesDao getSeriesSalesDao() {
102107
return new JdbcSeriesSalesDao(jdbcTemplate);
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
/*
2+
* Copyright (C) 2009-2017 Slava Semushin <[email protected]>
3+
*
4+
* This program is free software; you can redistribute it and/or modify
5+
* it under the terms of the GNU General Public License as published by
6+
* the Free Software Foundation; either version 2 of the License, or
7+
* (at your option) any later version.
8+
*
9+
* This program is distributed in the hope that it will be useful,
10+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
* GNU General Public License for more details.
13+
*
14+
* You should have received a copy of the GNU General Public License
15+
* along with this program; if not, write to the Free Software
16+
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17+
*/
18+
package ru.mystamps.web.config;
19+
20+
import java.util.HashMap;
21+
import java.util.List;
22+
import java.util.Map;
23+
24+
import javax.annotation.PostConstruct;
25+
26+
import org.apache.commons.lang3.StringUtils;
27+
28+
import org.slf4j.Logger;
29+
import org.slf4j.LoggerFactory;
30+
31+
import org.springframework.beans.factory.config.ConfigurableBeanFactory;
32+
import org.springframework.context.ApplicationEventPublisher;
33+
import org.springframework.context.ApplicationListener;
34+
import org.springframework.context.annotation.Bean;
35+
import org.springframework.context.annotation.Configuration;
36+
import org.springframework.core.env.ConfigurableEnvironment;
37+
import org.springframework.core.env.EnumerablePropertySource;
38+
import org.springframework.core.env.PropertySource;
39+
40+
import lombok.RequiredArgsConstructor;
41+
42+
// CheckStyle: ignore AvoidStarImportCheck for next 1 line
43+
import ru.mystamps.web.controller.event.*; // NOPMD: UnusedImports (false positive)
44+
import ru.mystamps.web.util.extractor.SiteParser;
45+
46+
@Configuration
47+
@RequiredArgsConstructor
48+
public class EventsConfig {
49+
50+
private static final Logger LOG = LoggerFactory.getLogger(EventsConfig.class);
51+
52+
private final ServicesConfig servicesConfig;
53+
private final ApplicationEventPublisher eventPublisher;
54+
private final ConfigurableBeanFactory beanFactory;
55+
private final ConfigurableEnvironment env;
56+
57+
@PostConstruct
58+
public void init() {
59+
Map<Integer, SiteParser> parsers = createSiteParsers();
60+
for (Map.Entry<Integer, SiteParser> entry : parsers.entrySet()) {
61+
Integer num = entry.getKey();
62+
SiteParser parser = entry.getValue();
63+
if (!parser.isFullyInitialized()) {
64+
LOG.warn("Ignored non-fully initialized site parser (app.site-parser[{}])", num);
65+
continue;
66+
}
67+
LOG.trace("Registering site parser for '{}'", parser);
68+
beanFactory.registerSingleton("siteParser" + num, parser);
69+
}
70+
}
71+
72+
@Bean
73+
public ApplicationListener<ImportRequestCreated> getImportRequestCreatedEventListener() {
74+
return new ImportRequestCreatedEventListener(
75+
LoggerFactory.getLogger(ImportRequestCreatedEventListener.class),
76+
servicesConfig.getSeriesDownloaderService(),
77+
servicesConfig.getSeriesImportService(),
78+
eventPublisher
79+
);
80+
}
81+
82+
@Bean
83+
public ApplicationListener<DownloadingFailed> getDownloadingFailedEventListener() {
84+
return new DownloadingFailedEventListener(
85+
LoggerFactory.getLogger(DownloadingFailedEventListener.class),
86+
servicesConfig.getSeriesImportService()
87+
);
88+
}
89+
90+
@Bean
91+
public ApplicationListener<DownloadingSucceeded> getDownloadingSucceededEventListener(
92+
List<SiteParser> siteParsers) {
93+
94+
return new DownloadingSucceededEventListener(
95+
LoggerFactory.getLogger(DownloadingSucceededEventListener.class),
96+
servicesConfig.getSeriesImportService(),
97+
siteParsers,
98+
eventPublisher
99+
);
100+
}
101+
102+
@Bean
103+
public ApplicationListener<ParsingFailed> getParsingFailedEventListener() {
104+
return new ParsingFailedEventListener(
105+
LoggerFactory.getLogger(ParsingFailedEventListener.class),
106+
servicesConfig.getSeriesImportService()
107+
);
108+
}
109+
110+
@SuppressWarnings("PMD.ModifiedCyclomaticComplexity") // TODO: deal with it someday
111+
private Map<Integer, SiteParser> createSiteParsers() {
112+
boolean foundSiteParserProps = false;
113+
Map<Integer, SiteParser> parsers = new HashMap<>();
114+
115+
for (PropertySource<?> source : env.getPropertySources()) {
116+
// while we expect that properties will be in PropertiesPropertySource, we use
117+
// EnumerablePropertySource to also handle systemProperties and systemEnvironment
118+
// that are MapPropertySource and SystemEnvironmentPropertySource respectively
119+
if (!(source instanceof EnumerablePropertySource<?>)) {
120+
LOG.trace("Ignored property source: {} ({})", source.getName(), source.getClass());
121+
continue;
122+
}
123+
124+
LOG.trace("Inspecting property source: {} ({})", source.getName(), source.getClass());
125+
126+
for (String name : ((EnumerablePropertySource<?>)source).getPropertyNames()) {
127+
if (!name.startsWith("app.site-parser")) {
128+
continue;
129+
}
130+
131+
String propertyValue = (String)source.getProperty(name);
132+
LOG.trace("Detected property '{}' with value {}", name, propertyValue);
133+
134+
// extract parser number (app.site-parser[2].name -> 2)
135+
String strNum = StringUtils.substringBetween(name, "[", "]");
136+
if (StringUtils.isBlank(strNum)) {
137+
LOG.warn("Ignored property '{}': could not extract index", name);
138+
continue;
139+
}
140+
141+
Integer num = Integer.valueOf(strNum);
142+
if (!parsers.containsKey(num)) {
143+
SiteParser parser =
144+
new SiteParser(); // NOPMD: AvoidInstantiatingObjectsInLoops
145+
parsers.put(num, parser);
146+
}
147+
148+
// extract parser property (app.site-parser[2].name -> name)
149+
String fieldName = StringUtils.substringAfterLast(name, ".");
150+
if (StringUtils.isBlank(fieldName)) {
151+
LOG.warn("Ignored property '{}': could not extract property name", name);
152+
continue;
153+
}
154+
155+
SiteParser parser = parsers.get(num);
156+
boolean validProperty = parser.setField(fieldName, propertyValue);
157+
if (!validProperty) {
158+
LOG.warn("Ignored property '{}': unknown or unsupported", name);
159+
continue;
160+
}
161+
foundSiteParserProps = true;
162+
}
163+
// we shouldn't process others to be able to override a property
164+
if (foundSiteParserProps) {
165+
break;
166+
}
167+
}
168+
169+
return parsers;
170+
}
171+
172+
}

src/main/java/ru/mystamps/web/config/MvcConfig.java

+4-1
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,10 @@
5252

5353
@Configuration
5454
@EnableScheduling
55-
@Import(ControllersConfig.class)
55+
@Import({
56+
ControllersConfig.class,
57+
EventsConfig.class,
58+
})
5659
@RequiredArgsConstructor
5760
public class MvcConfig extends WebMvcConfigurerAdapter {
5861

src/main/java/ru/mystamps/web/config/ServicesConfig.java

+25
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,13 @@ public DownloaderService getImageDownloaderService() {
9494
return new HttpURLConnectionDownloaderService(new String[]{"image/jpeg", "image/png"});
9595
}
9696

97+
@Bean
98+
public DownloaderService getSeriesDownloaderService() {
99+
return new HttpURLConnectionDownloaderService(
100+
new String[]{"text/html", "image/jpeg", "image/png"}
101+
);
102+
}
103+
97104
@Bean
98105
public ImageService getImageService() {
99106
return new ImageServiceImpl(
@@ -153,6 +160,24 @@ public SeriesService getSeriesService() {
153160
);
154161
}
155162

163+
@Bean
164+
public SeriesImportService getSeriesImportService() {
165+
return new SeriesImportServiceImpl(
166+
LoggerFactory.getLogger(SeriesImportServiceImpl.class),
167+
daoConfig.getSeriesImportDao(),
168+
getSeriesInfoExtractorService()
169+
);
170+
}
171+
172+
@Bean
173+
public SeriesInfoExtractorService getSeriesInfoExtractorService() {
174+
return new SeriesInfoExtractorServiceImpl(
175+
LoggerFactory.getLogger(SeriesInfoExtractorServiceImpl.class),
176+
getCategoryService(),
177+
getCountryService()
178+
);
179+
}
180+
156181
@Bean
157182
public SeriesSalesService getSeriesSalesService() {
158183
return new SeriesSalesServiceImpl(

src/main/java/ru/mystamps/web/controller/SeriesController.java

+7
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,7 @@ protected void initSeriesSalesFormBinder(WebDataBinder binder) {
130130
public void showForm(
131131
@Category @RequestParam(name = "category", required = false) LinkEntityDto category,
132132
@Country @RequestParam(name = "country", required = false) LinkEntityDto country,
133+
@RequestParam(name = "image_url", required = false) String imageUrl,
133134
Model model,
134135
Locale userLocale) {
135136

@@ -154,6 +155,12 @@ public void showForm(
154155
addSeriesForm.setCountry(country);
155156
}
156157

158+
if (imageUrl != null) {
159+
// in case user doesn't have permission to specify image URL,
160+
// field won't be shown on the page and this value will be ignored
161+
addSeriesForm.setImageUrl(imageUrl);
162+
}
163+
157164
model.addAttribute("addSeriesForm", addSeriesForm);
158165
}
159166

0 commit comments

Comments
 (0)