Matcher matcher = pattern.matcher(page.getUrl().toString());
String pageID = null;
while (matcher.find()) {
pageID = matcher.group(1);
}
if (pageID == null) {
return;
}
Map<String, Object> map = new HashMap<>();
String JsonContext = ReadFile
.ReadFile(System.getProperty("user.dir") + "/configuration/job_config.json");
JSONObject jsonObject = new JSONObject(JsonContext);
JSONArray wbsites = jsonObject.getJSONArray("wbsites");
for (int i = 0; i < wbsites.length(); i++) {
JSONObject wbsite = wbsites.getJSONObject(i);
String wbsitename = wbsite.getString("wbsitename");
map.put("resource", wbsitename);
JSONArray fields = wbsite.getJSONArray("fields");
for (int j = 0; j < fields.length(); j++) {
JSONObject field = fields.getJSONObject(j);
String chinesename = field.getString("chinesename");
String name = field.getString("name");
String path = field.getString("path");
if (path.startsWith("//")) {
String objectStr = Format_transform.gb2312ToUtf8(page.getHtml().xpath(path).toString());
System.out.print(objectStr);
map.put(name, Format_transform.change(objectStr));
} else {
String objectStr = Format_transform.gb2312ToUtf8(page.getHtml().regex(path).toString());
map.put(name, Format_transform.change(objectStr));
}
if (name.equals("companymess")) {
String companymess = Format_transform.gb2312ToUtf8(page.getHtml().xpath(path).toString());
String[] strs = UtilTools.parseCompony(companymess);
map.put("nature", Format_transform.change(strs[0]));
map.put("scale", Format_transform.change(strs[1]));
map.put("industry", Format_transform.change(strs[2]));
}
}
}
map.put("url", page.getUrl().toString());
if (pageID != null) {
map.put("id", pageID);
map.put(HDFS, job_rawdata_path + pageID + Suffix);
jobDataReposity.insertData("job_internet", map);
} else {
jobDataReposity.insertEndTime("job_internet", map);
}