diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java
index 18fd4d5c1d30eceb41f29813b6e1bbfeea661f16..57cf17be036e000a50586b62376c25c2acbe7c18 100755
--- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java
+++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java
@@ -651,7 +651,7 @@ private static CorrelationAttributeInstance makeCorrAttr(BlackboardArtifact arti
logger.log(Level.SEVERE, String.format("Error querying central repository (%s)", artifact), ex); // NON-NLS
return null;
} catch (CorrelationAttributeNormalizationException ex) {
- logger.log(Level.SEVERE, String.format("Error creating correlation attribute instance (%s)", artifact), ex); // NON-NLS
+ logger.log(Level.WARNING, String.format("Error creating correlation attribute instance (%s)", artifact)); // NON-NLS
return null;
} catch (NoCurrentCaseException ex) {
logger.log(Level.WARNING, "Error getting current case", ex); // NON-NLS
diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/NetworkUtils.java b/Core/src/org/sleuthkit/autopsy/coreutils/NetworkUtils.java
index 8078a8c6a32abe50083256a3e62f7e41a7730659..990757d1dbd4020c8bb1deca4180710e6c4e4622 100644
--- a/Core/src/org/sleuthkit/autopsy/coreutils/NetworkUtils.java
+++ b/Core/src/org/sleuthkit/autopsy/coreutils/NetworkUtils.java
@@ -24,6 +24,8 @@
import java.net.UnknownHostException;
import java.util.logging.Level;
import org.apache.commons.lang.StringUtils;
+import org.apache.commons.validator.routines.DomainValidator;
+import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeNormalizationException;
public class NetworkUtils {
@@ -113,14 +115,24 @@ public static String extractDomain(String urlString) {
} catch (MalformedURLException ex) {
//do not log if not a valid URL - we will try to extract it ourselves
}
-
- // if there is a valid url host, get base domain from that host
- // otherwise use urlString and parse the domain
+
String result = (StringUtils.isNotBlank(urlHost))
? getBaseDomain(urlHost)
: getBaseDomain(urlString);
- return result;
+ // if there is a valid url host, get base domain from that host
+ // otherwise use urlString and parse the domain
+ DomainValidator validator = DomainValidator.getInstance(true);
+ if (validator.isValid(result)) {
+ return result;
+ } else {
+ final String validIpAddressRegex = "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$";
+ if (result.matches(validIpAddressRegex)) {
+ return result;
+ } else {
+ return "";
+ }
+ }
}
}
diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chromium.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chromium.java
index 357f10b8a7908ba72624798c70fc0fa74544d204..c18d40fb00b574b39768871424c483195dfa6e5a 100644
--- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chromium.java
+++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chromium.java
@@ -43,6 +43,7 @@
import java.util.ArrayList;
import java.util.Arrays;
import org.apache.commons.io.FilenameUtils;
+import org.apache.commons.lang3.StringUtils;
import org.openide.util.NbBundle.Messages;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
@@ -253,26 +254,19 @@ private void getHistory(String browser, String browserLocation, long ingestJobId
tempList = this.querySQLiteDb(temps, HISTORY_QUERY);
logger.log(Level.INFO, "{0}- Now getting history from {1} with {2} artifacts identified.", new Object[]{getDisplayName(), temps, tempList.size()}); //NON-NLS
for (HashMap<String, Object> result : tempList) {
- Collection<BlackboardAttribute> bbattributes = new ArrayList<>();
- bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL,
- RecentActivityExtracterModuleFactory.getModuleName(),
- ((result.get("url").toString() != null) ? result.get("url").toString() : ""))); //NON-NLS
- bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED,
- RecentActivityExtracterModuleFactory.getModuleName(),
- (Long.valueOf(result.get("last_visit_time").toString()) / 1000000) - Long.valueOf("11644473600"))); //NON-NLS
- bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_REFERRER,
- RecentActivityExtracterModuleFactory.getModuleName(),
- ((result.get("from_visit").toString() != null) ? result.get("from_visit").toString() : ""))); //NON-NLS
- bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_TITLE,
- RecentActivityExtracterModuleFactory.getModuleName(),
- ((result.get("title").toString() != null) ? result.get("title").toString() : ""))); //NON-NLS
- bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME,
- RecentActivityExtracterModuleFactory.getModuleName(), browser));
- bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN,
- RecentActivityExtracterModuleFactory.getModuleName(),
- (NetworkUtils.extractDomain((result.get("url").toString() != null) ? result.get("url").toString() : "")))); //NON-NLS
-
+ String url = result.get("url") == null ? "" : result.get("url").toString();
+ String extractedDomain = NetworkUtils.extractDomain(url);
+
try {
+ Collection<BlackboardAttribute> bbattributes = createHistoryAttributes(
+ StringUtils.defaultString(url),
+ (Long.valueOf(result.get("last_visit_time").toString()) / 1000000) - Long.valueOf("11644473600"),
+ result.get("from_visit") == null ? "" : result.get("from_visit").toString(),
+ result.get("title") == null ? "" : result.get("title").toString(),
+ browser,
+ extractedDomain,
+ "");
+
bbartifacts.add(createArtifactWithAttributes(BlackboardArtifact.Type.TSK_WEB_HISTORY, historyFile, bbattributes));
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, String.format("Failed to create history artifact for file (%d)", historyFile.getId()), ex);
diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java
index 8eb649b93dbb2f1affbe6d7108add8945512d300..db509739438157fe74e6c6ed422d11cd3c92ca61 100644
--- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java
+++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java
@@ -35,8 +35,10 @@
import java.util.HashMap;
import java.util.List;
import java.util.logging.Level;
+import org.apache.commons.lang.StringUtils;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.coreutils.Logger;
+import org.sleuthkit.autopsy.coreutils.NetworkUtils;
import org.sleuthkit.autopsy.coreutils.SQLiteDBConnect;
import org.sleuthkit.autopsy.datamodel.ContentUtils;
import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress;
@@ -317,33 +319,44 @@ protected Collection<BlackboardAttribute> createHistoryAttributes(String url, Lo
Collection<BlackboardAttribute> bbattributes = new ArrayList<>();
bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_URL,
- RecentActivityExtracterModuleFactory.getModuleName(),
- (url != null) ? url : "")); //NON-NLS
+ RecentActivityExtracterModuleFactory.getModuleName(), url)); //NON-NLS
if (accessTime != null) {
bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED,
- RecentActivityExtracterModuleFactory.getModuleName(), accessTime));
+ RecentActivityExtracterModuleFactory.getModuleName(),
+ accessTime));
}
- bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_REFERRER,
- RecentActivityExtracterModuleFactory.getModuleName(),
- (referrer != null) ? referrer : "")); //NON-NLS
+ if (StringUtils.isNotBlank(referrer)) {
+ bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_REFERRER,
+ RecentActivityExtracterModuleFactory.getModuleName(),
+ referrer)); //NON-NLS
+ }
- bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_TITLE,
- RecentActivityExtracterModuleFactory.getModuleName(),
- (title != null) ? title : "")); //NON-NLS
+ if (StringUtils.isNotBlank(title)) {
+ bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_TITLE,
+ RecentActivityExtracterModuleFactory.getModuleName(),
+ title)); //NON-NLS
+ }
- bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PROG_NAME,
- RecentActivityExtracterModuleFactory.getModuleName(),
- (programName != null) ? programName : "")); //NON-NLS
+ if (StringUtils.isNotBlank(programName)) {
+ bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PROG_NAME,
+ RecentActivityExtracterModuleFactory.getModuleName(),
+ programName)); //NON-NLS
+ }
- bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DOMAIN,
- RecentActivityExtracterModuleFactory.getModuleName(),
- (domain != null) ? domain : "")); //NON-NLS
+
+ if (StringUtils.isNotBlank(url)) {
+ bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DOMAIN,
+ RecentActivityExtracterModuleFactory.getModuleName(),
+ domain)); //NON-NLS
+ }
- bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_USER_NAME,
- RecentActivityExtracterModuleFactory.getModuleName(),
- (user != null) ? user : "")); //NON-NLS
+ if (StringUtils.isNotBlank(user)) {
+ bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_USER_NAME,
+ RecentActivityExtracterModuleFactory.getModuleName(),
+ user)); //NON-NLS
+ }
return bbattributes;
}
diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java
index 41e5e0ceb36aa25dadd07b0dfbd6035eb868ce2a..370504ee95d5a13a89b6daeee082835c655f6c2f 100644
--- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java
+++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java
@@ -154,22 +154,16 @@ private void getBookmark() {
datetime = Long.valueOf(Tempdate);
String domain = extractDomain(url);
- Collection<BlackboardAttribute> bbattributes = new ArrayList<>();
- bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL,
- RecentActivityExtracterModuleFactory.getModuleName(), url));
- bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_TITLE,
- RecentActivityExtracterModuleFactory.getModuleName(), name));
- bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_CREATED,
- RecentActivityExtracterModuleFactory.getModuleName(), datetime));
- bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME,
- RecentActivityExtracterModuleFactory.getModuleName(),
- NbBundle.getMessage(this.getClass(), "ExtractIE.moduleName.text")));
- if (domain != null && domain.isEmpty() == false) {
- bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN,
- RecentActivityExtracterModuleFactory.getModuleName(), domain));
- }
-
try {
+ Collection<BlackboardAttribute> bbattributes = createHistoryAttributes(
+ url,
+ datetime,
+ null,
+ name,
+ NbBundle.getMessage(this.getClass(), "ExtractIE.moduleName.text"),
+ domain,
+ null);
+
bbartifacts.add(createArtifactWithAttributes(BlackboardArtifact.Type.TSK_WEB_BOOKMARK, fav, bbattributes));
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, String.format("Failed to create %s for file %d", ARTIFACT_TYPE.TSK_WEB_BOOKMARK.getDisplayName(), fav.getId()), ex);
@@ -567,28 +561,16 @@ private Collection<BlackboardArtifact> parsePascoOutput(AbstractFile origFile, S
}
}
- Collection<BlackboardAttribute> bbattributes = new ArrayList<>();
- bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL,
- RecentActivityExtracterModuleFactory.getModuleName(), realurl));
- //bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL_DECODED.getTypeID(), "RecentActivity", EscapeUtil.decodeURL(realurl)));
-
- bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED,
- RecentActivityExtracterModuleFactory.getModuleName(), ftime));
- bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_REFERRER,
- RecentActivityExtracterModuleFactory.getModuleName(), ""));
- // @@@ NOte that other browser modules are adding TITLE in here for the title
- bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME,
- RecentActivityExtracterModuleFactory.getModuleName(),
- NbBundle.getMessage(this.getClass(),
- "ExtractIE.moduleName.text")));
- if (domain != null && domain.isEmpty() == false) {
- bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN,
- RecentActivityExtracterModuleFactory.getModuleName(), domain));
- }
- bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_USER_NAME,
- RecentActivityExtracterModuleFactory.getModuleName(), user));
-
try {
+ Collection<BlackboardAttribute> bbattributes = createHistoryAttributes(
+ realurl,
+ ftime,
+ null,
+ null,
+ NbBundle.getMessage(this.getClass(), "ExtractIE.moduleName.text"),
+ domain,
+ user);
+
bbartifacts.add(createArtifactWithAttributes(BlackboardArtifact.Type.TSK_WEB_HISTORY, origFile, bbattributes));
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, String.format("Failed to create %s for file %d", BlackboardArtifact.Type.TSK_WEB_HISTORY.getDisplayName(), origFile.getId()), ex);
diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java
index 163feec22dfe5c5de82fbc483e8c04c3f5b9f9a2..27bc0177acbafed2c922d09c0e4e2d5eedeff442 100644
--- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java
+++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java
@@ -217,32 +217,19 @@ private void getHistory(long ingestJobId) {
}
String url = result.get("url").toString();
-
- Collection<BlackboardAttribute> bbattributes = new ArrayList<>();
- bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL,
- RecentActivityExtracterModuleFactory.getModuleName(),
- ((url != null) ? url : ""))); //NON-NLS
- //bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL_DECODED.getTypeID(), "RecentActivity", ((result.get("url").toString() != null) ? EscapeUtil.decodeURL(result.get("url").toString()) : "")));
- bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED,
- RecentActivityExtracterModuleFactory.getModuleName(),
- (Long.valueOf(result.get("visit_date").toString())))); //NON-NLS
- bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_REFERRER,
- RecentActivityExtracterModuleFactory.getModuleName(),
- ((result.get("ref").toString() != null) ? result.get("ref").toString() : ""))); //NON-NLS
- bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_TITLE,
- RecentActivityExtracterModuleFactory.getModuleName(),
- ((result.get("title").toString() != null) ? result.get("title").toString() : ""))); //NON-NLS
- bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME,
- RecentActivityExtracterModuleFactory.getModuleName(),
- NbBundle.getMessage(this.getClass(), "Firefox.moduleName")));
String domain = extractDomain(url);
- if (domain != null && domain.isEmpty() == false) {
- bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN,
- RecentActivityExtracterModuleFactory.getModuleName(), domain)); //NON-NLS
-
- }
-
try {
+
+ Collection<BlackboardAttribute> bbattributes = createHistoryAttributes(
+ url,
+ Long.valueOf(result.get("visit_date").toString()),
+ result.get("ref").toString(),
+ result.get("title").toString(),
+ NbBundle.getMessage(this.getClass(), "Firefox.moduleName"),
+ domain,
+ null);
+
+
bbartifacts.add(createArtifactWithAttributes(BlackboardArtifact.Type.TSK_WEB_HISTORY, historyFile, bbattributes));
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, String.format("Failed to create TSK_WEB_HISTORY artifact for file %d", historyFile.getId()), ex);