Skip to content

Commit cb4cc66

Browse files
php-coderphp-coder
committed
refactor: add code for migrating existing site parsers from application.properties file to database.
Addressed to #975
1 parent d476244 commit cb4cc66

15 files changed

+574
-2
lines changed

‎src/main/java/ru/mystamps/web/config/ApplicationContext.java

+1
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ public static PropertySourcesPlaceholderConfigurer getPropertySourcesPlaceholder
5252
new ClassPathResource("sql/series_dao_queries.properties"),
5353
new ClassPathResource("sql/series_import_request_dao_queries.properties"),
5454
new ClassPathResource("sql/series_sales_dao_queries.properties"),
55+
new ClassPathResource("sql/site_parser_dao_queries.properties"),
5556
new ClassPathResource("sql/suspicious_activity_dao_queries.properties"),
5657
new ClassPathResource("sql/transaction_participants_dao_queries.properties")
5758
);

‎src/main/java/ru/mystamps/web/feature/series/importing/event/DownloadingSucceededEventListener.java

+37
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,11 @@
3030

3131
import ru.mystamps.web.feature.series.importing.RawParsedDataDto;
3232
import ru.mystamps.web.feature.series.importing.SeriesImportService;
33+
import ru.mystamps.web.feature.series.importing.extractor.JsoupSiteParser;
3334
import ru.mystamps.web.feature.series.importing.extractor.SeriesInfo;
3435
import ru.mystamps.web.feature.series.importing.extractor.SiteParser;
36+
import ru.mystamps.web.feature.series.importing.extractor.SiteParserConfiguration;
37+
import ru.mystamps.web.feature.series.importing.extractor.SiteParserService;
3538

3639
/**
3740
* Listener of the {@link DownloadingSucceeded} event.
@@ -49,12 +52,16 @@ public class DownloadingSucceededEventListener
4952

5053
private final Logger log;
5154
private final SeriesImportService seriesImportService;
55+
private final SiteParserService siteParserService;
5256
private final List<SiteParser> siteParsers;
5357
private final ApplicationEventPublisher eventPublisher;
5458

5559
@PostConstruct
5660
public void init() {
5761
log.info("Registered site parsers: {}", siteParsers);
62+
63+
// TODO: remove migration logic after finishing migration
64+
siteParsers.forEach(this::migrateParser);
5865
}
5966

6067
@Override
@@ -70,6 +77,7 @@ public void onApplicationEvent(DownloadingSucceeded event) {
7077
return;
7178
}
7279

80+
// TODO: replace with siteParserService.findForUrl(url) and update diagrams
7381
String url = event.getUrl();
7482
SiteParser parser = null;
7583
for (SiteParser candidate : siteParsers) {
@@ -107,4 +115,33 @@ public void onApplicationEvent(DownloadingSucceeded event) {
107115
seriesImportService.saveParsedData(requestId, data);
108116
}
109117

118+
private void migrateParser(SiteParser parser) {
119+
if (!(parser instanceof JsoupSiteParser)) {
120+
log.warn("Could not migrate unknown (non-Jsoup based) parser: {}", parser);
121+
return;
122+
}
123+
124+
SiteParserConfiguration cfg = ((JsoupSiteParser)parser).toConfiguration();
125+
String url = cfg.getMatchedUrl();
126+
String name = cfg.getName();
127+
if (siteParserService.findForUrl(url) != null) {
128+
log.warn(
129+
"Parser '{}': already exist in database and "
130+
+ "can be removed from application*.properties file",
131+
name
132+
);
133+
return;
134+
}
135+
136+
log.info("Parser '{}': migrating to database", name);
137+
138+
siteParserService.add(cfg);
139+
140+
log.warn(
141+
"Parser '{}': successfully migrated and "
142+
+ "can be removed from application*.properties file",
143+
name
144+
);
145+
}
146+
110147
}

‎src/main/java/ru/mystamps/web/feature/series/importing/event/EventsConfig.java

+28-1
Original file line numberDiff line numberDiff line change
@@ -35,16 +35,22 @@
3535
import org.springframework.context.ApplicationListener;
3636
import org.springframework.context.annotation.Bean;
3737
import org.springframework.context.annotation.Configuration;
38+
import org.springframework.context.annotation.DependsOn;
3839
import org.springframework.core.env.ConfigurableEnvironment;
3940
import org.springframework.core.env.EnumerablePropertySource;
4041
import org.springframework.core.env.PropertySource;
42+
import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
4143

4244
import lombok.RequiredArgsConstructor;
4345

4446
import ru.mystamps.web.config.ServicesConfig;
4547
import ru.mystamps.web.feature.series.importing.SeriesImportService;
48+
import ru.mystamps.web.feature.series.importing.extractor.JdbcSiteParserDao;
4649
import ru.mystamps.web.feature.series.importing.extractor.JsoupSiteParser;
4750
import ru.mystamps.web.feature.series.importing.extractor.SiteParser;
51+
import ru.mystamps.web.feature.series.importing.extractor.SiteParserDao;
52+
import ru.mystamps.web.feature.series.importing.extractor.SiteParserService;
53+
import ru.mystamps.web.feature.series.importing.extractor.SiteParserServiceImpl;
4854
import ru.mystamps.web.feature.series.importing.extractor.TimedSiteParser;
4955

5056
@Configuration
@@ -58,6 +64,7 @@ public class EventsConfig {
5864
private final ApplicationEventPublisher eventPublisher;
5965
private final ConfigurableBeanFactory beanFactory;
6066
private final ConfigurableEnvironment env;
67+
private final NamedParameterJdbcTemplate jdbcTemplate;
6168

6269
@PostConstruct
6370
public void init() {
@@ -74,6 +81,19 @@ public void init() {
7481
}
7582
}
7683

84+
@Bean
85+
public SiteParserService siteParserService(SiteParserDao siteParserDao) {
86+
return new SiteParserServiceImpl(
87+
LoggerFactory.getLogger(SiteParserServiceImpl.class),
88+
siteParserDao
89+
);
90+
}
91+
92+
@Bean
93+
public SiteParserDao siteParserDao() {
94+
return new JdbcSiteParserDao(jdbcTemplate);
95+
}
96+
7797
@Bean
7898
public ApplicationListener<ImportRequestCreated> getImportRequestCreatedEventListener() {
7999
return new ImportRequestCreatedEventListener(
@@ -84,13 +104,20 @@ public ApplicationListener<ImportRequestCreated> getImportRequestCreatedEventLis
84104
);
85105
}
86106

107+
// This bean has logic that modifies database. To ensure that all migrations have been applied
108+
// we need this dependency. This annotation shouldn't be needed in Spring Boot 2.
109+
// TODO: remove this annotation when migration will be completed
110+
@DependsOn("liquibase")
87111
@Bean
88112
public ApplicationListener<DownloadingSucceeded> getDownloadingSucceededEventListener(
89-
Optional<List<SiteParser>> siteParsers) {
113+
Optional<List<SiteParser>> siteParsers,
114+
SiteParserService siteParserService
115+
) {
90116

91117
return new DownloadingSucceededEventListener(
92118
LoggerFactory.getLogger(DownloadingSucceededEventListener.class),
93119
seriesImportService,
120+
siteParserService,
94121
siteParsers.orElse(Collections.emptyList()),
95122
eventPublisher
96123
);

‎src/main/java/ru/mystamps/web/feature/series/importing/extractor/AddParserParameterDbDto.java

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
/*
2+
* Copyright (C) 2009-2018 Slava Semushin <[email protected]>
3+
*
4+
* This program is free software; you can redistribute it and/or modify
5+
* it under the terms of the GNU General Public License as published by
6+
* the Free Software Foundation; either version 2 of the License, or
7+
* (at your option) any later version.
8+
*
9+
* This program is distributed in the hope that it will be useful,
10+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
* GNU General Public License for more details.
13+
*
14+
* You should have received a copy of the GNU General Public License
15+
* along with this program; if not, write to the Free Software
16+
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17+
*/
18+
package ru.mystamps.web.feature.series.importing.extractor;
19+
20+
import lombok.Getter;
21+
import lombok.RequiredArgsConstructor;
22+
23+
@Getter
24+
@RequiredArgsConstructor
25+
public class AddParserParameterDbDto {
26+
private final Integer parserId;
27+
private final String name;
28+
private final String value;
29+
}

‎src/main/java/ru/mystamps/web/feature/series/importing/extractor/JdbcSiteParserDao.java

+116
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
/*
2+
* Copyright (C) 2009-2018 Slava Semushin <[email protected]>
3+
*
4+
* This program is free software; you can redistribute it and/or modify
5+
* it under the terms of the GNU General Public License as published by
6+
* the Free Software Foundation; either version 2 of the License, or
7+
* (at your option) any later version.
8+
*
9+
* This program is distributed in the hope that it will be useful,
10+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
* GNU General Public License for more details.
13+
*
14+
* You should have received a copy of the GNU General Public License
15+
* along with this program; if not, write to the Free Software
16+
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17+
*/
18+
package ru.mystamps.web.feature.series.importing.extractor;
19+
20+
import java.util.Collections;
21+
import java.util.HashMap;
22+
import java.util.Map;
23+
24+
import org.apache.commons.lang3.Validate;
25+
26+
import org.springframework.beans.factory.annotation.Value;
27+
import org.springframework.dao.EmptyResultDataAccessException;
28+
import org.springframework.jdbc.core.ResultSetExtractor;
29+
import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
30+
import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
31+
import org.springframework.jdbc.support.GeneratedKeyHolder;
32+
import org.springframework.jdbc.support.KeyHolder;
33+
34+
import lombok.RequiredArgsConstructor;
35+
36+
@RequiredArgsConstructor
37+
public class JdbcSiteParserDao implements SiteParserDao {
38+
39+
private static final ResultSetExtractor<Map<String, String>> PARAMS_EXTRACTOR =
40+
new MapResultSetExtractor("name", "value");
41+
42+
private final NamedParameterJdbcTemplate jdbcTemplate;
43+
44+
@Value("${site_parser.create}")
45+
private String addParserSql;
46+
47+
@Value("${site_parser_param.create}")
48+
private String addParserParameterSql;
49+
50+
@Value("${site_parser.find_like_matched_url}")
51+
private String findParserIdByMatchedUrl;
52+
53+
@SuppressWarnings("PMD.LongVariable")
54+
@Value("${site_parser_param.find_all_with_parser_name}")
55+
private String findParametersWithParserNameSql;
56+
57+
@Override
58+
public Integer addParser(String name) {
59+
KeyHolder holder = new GeneratedKeyHolder();
60+
61+
int affected = jdbcTemplate.update(
62+
addParserSql,
63+
new MapSqlParameterSource("name", name),
64+
holder
65+
);
66+
67+
Validate.validState(
68+
affected == 1,
69+
"Unexpected number of affected rows after creation of site parser: %d",
70+
affected
71+
);
72+
73+
return holder.getKey().intValue();
74+
}
75+
76+
@Override
77+
public void addParserParameter(AddParserParameterDbDto param) {
78+
Map<String, Object> params = new HashMap<>();
79+
params.put("parser_id", param.getParserId());
80+
params.put("name", param.getName());
81+
params.put("value", param.getValue());
82+
83+
int affected = jdbcTemplate.update(addParserParameterSql, params);
84+
85+
Validate.validState(
86+
affected == 1,
87+
"Unexpected number of affected rows after adding parser parameter: %d",
88+
affected
89+
);
90+
}
91+
92+
@Override
93+
public Integer findParserIdForUrl(String url) {
94+
try {
95+
return jdbcTemplate.queryForObject(
96+
findParserIdByMatchedUrl,
97+
Collections.singletonMap("url", url),
98+
Integer.class
99+
);
100+
} catch (EmptyResultDataAccessException ignored) {
101+
return null;
102+
}
103+
}
104+
105+
@Override
106+
public SiteParserConfiguration findConfigurationForParser(Integer parserId) {
107+
Map<String, String> params = jdbcTemplate.query(
108+
findParametersWithParserNameSql,
109+
Collections.singletonMap("parser_id", parserId),
110+
PARAMS_EXTRACTOR
111+
);
112+
113+
return new SiteParserConfiguration(params);
114+
}
115+
116+
}

‎src/main/java/ru/mystamps/web/feature/series/importing/extractor/JsoupSiteParser.java

+22-1
Original file line numberDiff line numberDiff line change
@@ -35,14 +35,20 @@
3535
// Getters/setters are being used in unit tests
3636
@Getter(AccessLevel.PROTECTED)
3737
@Setter(AccessLevel.PROTECTED)
38-
@SuppressWarnings("PMD.TooManyMethods")
38+
@SuppressWarnings({
39+
"PMD.GodClass",
40+
"PMD.TooManyMethods",
41+
// false positive because setField() modifies the fields
42+
"PMD.ImmutableField"
43+
})
3944
public class JsoupSiteParser implements SiteParser {
4045
private static final Logger LOG = LoggerFactory.getLogger(JsoupSiteParser.class);
4146

4247
// When you're adding a new field don't forget to also update:
4348
// - JsoupSiteParser.setField()
4449
// - JsoupSiteParser.isFullyInitialized() (optionally)
4550
// - JsoupSiteParserTest.describe()
51+
// - SiteParserConfiguration
4652
private String name;
4753
private String matchedUrl;
4854
private String categoryLocator;
@@ -55,6 +61,21 @@ public class JsoupSiteParser implements SiteParser {
5561
private String priceLocator;
5662
private String currencyValue;
5763

64+
// TODO: remove because it's only needed for migrating configuration from file to database
65+
public SiteParserConfiguration toConfiguration() {
66+
SiteParserConfiguration cfg = new SiteParserConfiguration(name, matchedUrl);
67+
cfg.setCategoryLocator(categoryLocator);
68+
cfg.setCountryLocator(countryLocator);
69+
cfg.setShortDescriptionLocator(shortDescriptionLocator);
70+
cfg.setImageUrlLocator(imageUrlLocator);
71+
cfg.setImageUrlAttribute(imageUrlAttribute);
72+
cfg.setIssueDateLocator(issueDateLocator);
73+
cfg.setSellerLocator(sellerLocator);
74+
cfg.setPriceLocator(priceLocator);
75+
cfg.setCurrencyValue(currencyValue);
76+
return cfg;
77+
}
78+
5879
@Override
5980
public boolean setField(String name, String value) {
6081
Validate.validState(StringUtils.isNotBlank(name), "Field name must be non-blank");

‎src/main/java/ru/mystamps/web/feature/series/importing/extractor/MapResultSetExtractor.java

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
/*
2+
* Copyright (C) 2009-2018 Slava Semushin <[email protected]>
3+
*
4+
* This program is free software; you can redistribute it and/or modify
5+
* it under the terms of the GNU General Public License as published by
6+
* the Free Software Foundation; either version 2 of the License, or
7+
* (at your option) any later version.
8+
*
9+
* This program is distributed in the hope that it will be useful,
10+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
* GNU General Public License for more details.
13+
*
14+
* You should have received a copy of the GNU General Public License
15+
* along with this program; if not, write to the Free Software
16+
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17+
*/
18+
package ru.mystamps.web.feature.series.importing.extractor;
19+
20+
import java.sql.ResultSet;
21+
import java.sql.SQLException;
22+
import java.util.HashMap;
23+
import java.util.Map;
24+
25+
import org.springframework.dao.DataAccessException;
26+
import org.springframework.jdbc.core.ResultSetExtractor;
27+
28+
import lombok.RequiredArgsConstructor;
29+
30+
@RequiredArgsConstructor
31+
class MapResultSetExtractor implements ResultSetExtractor<Map<String, String>> {
32+
33+
private final String keyFieldName;
34+
private final String valueFieldName;
35+
36+
@Override
37+
public Map<String, String> extractData(ResultSet rs) throws SQLException, DataAccessException {
38+
Map<String, String> result = new HashMap<>();
39+
40+
while (rs.next()) {
41+
String key = rs.getString(keyFieldName);
42+
String value = rs.getString(valueFieldName);
43+
result.put(key, value);
44+
}
45+
46+
return result;
47+
}
48+
49+
}

0 commit comments

Comments
 (0)