Skip to content
Snippets Groups Projects
Commit 1b78b496 authored by Greg DiCristofaro's avatar Greg DiCristofaro
Browse files

updates for path normalizer

parent 3f2f6a45
No related branches found
No related tags found
No related merge requests found
...@@ -136,7 +136,6 @@ private static class SharedProcessing { ...@@ -136,7 +136,6 @@ private static class SharedProcessing {
private final CTLicensePersistence ctSettingsPersistence = CTLicensePersistence.getInstance(); private final CTLicensePersistence ctSettingsPersistence = CTLicensePersistence.getInstance();
private final CTApiDAO ctApiDAO = CTApiDAO.getInstance(); private final CTApiDAO ctApiDAO = CTApiDAO.getInstance();
private final UsernameAnonymizer usernameAnonymizer = new UsernameAnonymizer();
private IngestJobState ingestJobState = null; private IngestJobState ingestJobState = null;
...@@ -235,6 +234,7 @@ private IngestJobState getNewJobState(IngestJobContext context) throws Exception ...@@ -235,6 +234,7 @@ private IngestJobState getNewJobState(IngestJobContext context) throws Exception
return new IngestJobState( return new IngestJobState(
context, context,
tskCase, tskCase,
new PathNormalizer(tskCase),
new FileTypeDetector(), new FileTypeDetector(),
licenseInfoOpt.get(), licenseInfoOpt.get(),
malwareType, malwareType,
...@@ -657,7 +657,7 @@ private boolean uploadFile(IngestJobState ingestJobState, String md5, long objId ...@@ -657,7 +657,7 @@ private boolean uploadFile(IngestJobState ingestJobState, String md5, long objId
// upload metadata // upload metadata
MetadataUploadRequest metaRequest = new MetadataUploadRequest() MetadataUploadRequest metaRequest = new MetadataUploadRequest()
.setCreatedDate(af.getCrtime() == 0 ? null : af.getCrtime()) .setCreatedDate(af.getCrtime() == 0 ? null : af.getCrtime())
.setFilePath(usernameAnonymizer.anonymousUsername(af.getUniquePath())) .setFilePath(ingestJobState.getPathNormalizer().normalizePath(af.getUniquePath()))
.setFileSizeBytes(af.getSize()) .setFileSizeBytes(af.getSize())
.setFileUploadUrl(authTokenResponse.getFileUploadUrl()) .setFileUploadUrl(authTokenResponse.getFileUploadUrl())
.setMd5(md5) .setMd5(md5)
...@@ -733,7 +733,7 @@ private void longPollForNotFound(IngestJobState ingestJobState) throws Interrupt ...@@ -733,7 +733,7 @@ private void longPollForNotFound(IngestJobState ingestJobState) throws Interrupt
if (!ingestJobState.isDoFileLookups() || ingestJobState.getIngestJobContext().fileIngestIsCancelled()) { if (!ingestJobState.isDoFileLookups() || ingestJobState.getIngestJobContext().fileIngestIsCancelled()) {
return; return;
} }
Thread.sleep(FILE_UPLOAD_RETRY_SLEEP_MILLIS); Thread.sleep(FILE_UPLOAD_RETRY_SLEEP_MILLIS);
} }
} }
...@@ -934,6 +934,7 @@ static class IngestJobState { ...@@ -934,6 +934,7 @@ static class IngestJobState {
null, null,
null, null,
null, null,
null,
false, false,
false false
); );
...@@ -951,10 +952,12 @@ static class IngestJobState { ...@@ -951,10 +952,12 @@ static class IngestJobState {
private boolean uploadUnknownFiles; private boolean uploadUnknownFiles;
private boolean doFileLookups; private boolean doFileLookups;
private final IngestJobContext ingestJobContext; private final IngestJobContext ingestJobContext;
private final PathNormalizer pathNormalizer;
IngestJobState(IngestJobContext ingestJobContext, SleuthkitCase tskCase, FileTypeDetector fileTypeDetector, LicenseInfo licenseInfo, BlackboardArtifact.Type malwareType, boolean uploadUnknownFiles, boolean doFileLookups) { IngestJobState(IngestJobContext ingestJobContext, SleuthkitCase tskCase, PathNormalizer pathNormalizer, FileTypeDetector fileTypeDetector, LicenseInfo licenseInfo, BlackboardArtifact.Type malwareType, boolean uploadUnknownFiles, boolean doFileLookups) {
this.tskCase = tskCase; this.tskCase = tskCase;
this.fileTypeDetector = fileTypeDetector; this.fileTypeDetector = fileTypeDetector;
this.pathNormalizer = pathNormalizer;
this.licenseInfo = licenseInfo; this.licenseInfo = licenseInfo;
this.malwareType = malwareType; this.malwareType = malwareType;
this.dsId = ingestJobContext == null ? 0L : ingestJobContext.getDataSource().getId(); this.dsId = ingestJobContext == null ? 0L : ingestJobContext.getDataSource().getId();
...@@ -1017,6 +1020,11 @@ boolean isDoFileLookups() { ...@@ -1017,6 +1020,11 @@ boolean isDoFileLookups() {
void disableDoFileLookups() { void disableDoFileLookups() {
this.doFileLookups = false; this.doFileLookups = false;
} }
public PathNormalizer getPathNormalizer() {
return pathNormalizer;
}
} }
} }
} }
...@@ -20,42 +20,64 @@ ...@@ -20,42 +20,64 @@
import com.google.common.net.InetAddresses; import com.google.common.net.InetAddresses;
import java.net.InetAddress; import java.net.InetAddress;
import java.util.Collections;
import java.util.HashSet; import java.util.HashSet;
import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.Set; import java.util.Set;
import java.util.logging.Level; import java.util.logging.Level;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
/** /**
* Utility class to anonymize username in paths also anonymizes hostname / ip * Utility class to anonymize paths.
* from UNC paths
*/ */
class UsernameAnonymizer { class PathNormalizer {
private static final Logger LOGGER = Logger.getLogger(UsernameAnonymizer.class.getName()); private static final Logger LOGGER = Logger.getLogger(PathNormalizer.class.getName());
private final String USER_PATH_FORWARD_SLASH_REGEX = "(?<!all )([/]{0,1}\\Qusers\\E/)(?!(public|Default|defaultAccount|All Users))([^/]+)(/){0,1}"; private static final String ANONYMIZED_USERNAME = "<user>";
private final String USER_PATH_BACK_SLASH_REGEX = "(?<!all )([\\\\]{0,1}\\Qusers\\E\\\\)(?!(public|Default|defaultAccount|All Users))([^\\\\]+)([\\\\]){0,1}"; private static final String ANONYMIZED_IP = "<private_ip>";
private static final String ANONYMIZED_HOSTNAME = "<hostname>";
private static final String FORWARD_SLASH = "/";
private static final String BACK_SLASH = "\\";
private final double WINDOWS_VERSION; private static final Pattern USER_PATH_FORWARD_SLASH_REGEX = Pattern.compile("(?<!all )([/]{0,1}\\Qusers\\E/)(?!(public|Default|defaultAccount|All Users))([^/]+)(/){0,1}", Pattern.CASE_INSENSITIVE);
private final double DEFAULT_WINDOWS_VERSION = 10.0; private static final Pattern USER_PATH_BACK_SLASH_REGEX = Pattern.compile("(?<!all )([\\\\]{0,1}\\Qusers\\E\\\\)(?!(public|Default|defaultAccount|All Users))([^\\\\]+)([\\\\]){0,1}", Pattern.CASE_INSENSITIVE);
private final String USER_PATH_FORWARD_SLASH_REGEX_XP = "([/]{0,1}\\Qdocuments and settings\\E/)(?!(Default User|All Users))([^/]+)(/){0,1}";
private final String USER_PATH_BACK_SLASH_REGEX_XP = "([\\\\]{0,1}\\Qdocuments and settings\\E\\\\)(?!(Default User|All Users))([^\\\\]+)(\\\\){0,1}";
private final Pattern UNC_PATH_FORWARD_SLASH_PATTERN = Pattern.compile("(//)([^/]+)(/){0,1}"); private static final Pattern USER_PATH_FORWARD_SLASH_REGEX_XP = Pattern.compile("([/]{0,1}\\Qdocuments and settings\\E/)(?!(Default User|All Users))([^/]+)(/){0,1}", Pattern.CASE_INSENSITIVE);
private final Pattern UNC_PATH_BACK_SLASH_PATTERN = Pattern.compile("(\\\\\\\\)([^\\\\]+)(\\\\){0,1}"); private static final Pattern USER_PATH_BACK_SLASH_REGEX_XP = Pattern.compile("([\\\\]{0,1}\\Qdocuments and settings\\E\\\\)(?!(Default User|All Users))([^\\\\]+)(\\\\){0,1}", Pattern.CASE_INSENSITIVE);
public UsernameAnonymizer() { private static final Pattern UNC_PATH_FORWARD_SLASH_PATTERN = Pattern.compile("(//)([^/]+)(/){0,1}");
// This constructor was added for the unit tests private static final Pattern UNC_PATH_BACK_SLASH_PATTERN = Pattern.compile("(\\\\\\\\)([^\\\\]+)(\\\\){0,1}");
// For most purposes, the other constructor should be used so we get the collection info such as users and windows version
WINDOWS_VERSION = DEFAULT_WINDOWS_VERSION; private static final String USERNAME_REGEX_REPLACEMENT = "$1" + ANONYMIZED_USERNAME + "$4";
private final SleuthkitCase skCase;
PathNormalizer(SleuthkitCase skCase) {
this.skCase = skCase;
}
protected List<String> getUsernames() {
try {
return this.skCase.getOsAccountManager().getOsAccounts().stream()
.filter(acct -> acct != null)
.map(acct -> acct.getLoginName().orElse(null))
.filter(StringUtils::isNotBlank)
.collect(Collectors.toList());
} catch (TskCoreException ex) {
LOGGER.log(Level.WARNING, "There was an error getting current os accounts", ex);
return Collections.emptyList();
}
} }
public String anonymousUsername(String inputString) { public String normalizePath(String inputString) {
if (StringUtils.isBlank(inputString)) { if (StringUtils.isBlank(inputString)) {
return ""; return "";
} }
...@@ -68,27 +90,20 @@ public String anonymousUsername(String inputString) { ...@@ -68,27 +90,20 @@ public String anonymousUsername(String inputString) {
} }
private String anonymizeUserFromPathsWithForwardSlashes(String stringWithUsername) { private String anonymizeUserFromPathsWithForwardSlashes(String stringWithUsername) {
Pattern pattern = WINDOWS_VERSION < 6 ? Pattern.compile(USER_PATH_FORWARD_SLASH_REGEX_XP, Pattern.CASE_INSENSITIVE) : Pattern.compile(USER_PATH_FORWARD_SLASH_REGEX, Pattern.CASE_INSENSITIVE); String anonymousString = stringWithUsername;
Matcher matcher = pattern.matcher(stringWithUsername.toLowerCase(Locale.ENGLISH)); anonymousString = regexReplace(anonymousString, USER_PATH_FORWARD_SLASH_REGEX_XP, USERNAME_REGEX_REPLACEMENT);
String replacement = ""; anonymousString = regexReplace(anonymousString, USER_PATH_FORWARD_SLASH_REGEX, USERNAME_REGEX_REPLACEMENT);
while (matcher.find()) { anonymousString = replaceFolder(anonymousString, getUsernames(), ANONYMIZED_USERNAME, FORWARD_SLASH);
replacement = String.format("$1%s$4", "<user>");
}
String anonymousString = matcher.replaceAll(replacement);
return anonymousString; return anonymousString;
} }
// Most paths in CyberTriage are normalized with forward slashes // Most paths in CyberTriage are normalized with forward slashes
// but there can still be strings containing paths that are not normalized such paths contained in arguments or event log payloads // but there can still be strings containing paths that are not normalized such paths contained in arguments or event log payloads
private String anonymizeUserFromPathsWithBackSlashes(String stringWithUsername) { private String anonymizeUserFromPathsWithBackSlashes(String stringWithUsername) {
Pattern pattern = WINDOWS_VERSION < 6 ? Pattern.compile(USER_PATH_BACK_SLASH_REGEX_XP, Pattern.CASE_INSENSITIVE) : Pattern.compile(USER_PATH_BACK_SLASH_REGEX, Pattern.CASE_INSENSITIVE); String anonymousString = stringWithUsername;
Matcher matcher = pattern.matcher(stringWithUsername.toLowerCase(Locale.ENGLISH)); anonymousString = regexReplace(anonymousString, USER_PATH_BACK_SLASH_REGEX_XP, USERNAME_REGEX_REPLACEMENT);
String replacement = ""; anonymousString = regexReplace(anonymousString, USER_PATH_BACK_SLASH_REGEX, USERNAME_REGEX_REPLACEMENT);
while (matcher.find()) { anonymousString = replaceFolder(anonymousString, getUsernames(), ANONYMIZED_USERNAME, BACK_SLASH);
replacement = String.format("$1%s$4", "<user>");
}
String anonymousString = matcher.replaceAll(replacement);
return anonymousString; return anonymousString;
} }
...@@ -116,14 +131,10 @@ private String anonymizeServerFromUNCPath(String inputString) { ...@@ -116,14 +131,10 @@ private String anonymizeServerFromUNCPath(String inputString) {
continue; continue;
} }
if (InetAddresses.isInetAddress(serverName)) { if (InetAddresses.isInetAddress(serverName) && isLocalIP(serverName)) {
if (isLocalIP(serverName)) { anonymousString = replaceFolder(anonymousString, Collections.singletonList(serverName), ANONYMIZED_IP);
anonymousString = StringUtils.replace(anonymousString, "\\" + serverName + "\\", "\\<private_ip>\\");
anonymousString = StringUtils.replace(anonymousString, "/" + serverName + "/", "/<private_ip>/");
}
} else { } else {
anonymousString = StringUtils.replace(anonymousString, "\\" + serverName + "\\", "\\<hostname>\\"); anonymousString = replaceFolder(anonymousString, Collections.singletonList(serverName), ANONYMIZED_HOSTNAME);
anonymousString = StringUtils.replace(anonymousString, "/" + serverName + "/", "/<hostname>/");
} }
} }
...@@ -131,6 +142,41 @@ private String anonymizeServerFromUNCPath(String inputString) { ...@@ -131,6 +142,41 @@ private String anonymizeServerFromUNCPath(String inputString) {
return anonymousString; return anonymousString;
} }
private static String regexReplace(String orig, Pattern pattern, String regexReplacement) {
Matcher matcher = pattern.matcher(orig);
return matcher.replaceAll(regexReplacement);
}
private static String replaceFolder(String orig, List<String> valuesToReplace, String replacementValue) {
String anonymized = orig;
anonymized = replaceFolder(anonymized, valuesToReplace, replacementValue, FORWARD_SLASH);
anonymized = replaceFolder(anonymized, valuesToReplace, replacementValue, BACK_SLASH);
return anonymized;
}
private static String replaceFolder(String orig, List<String> valuesToReplace, String replacementValue, String folderDelimiter) {
if (orig == null || valuesToReplace == null) {
return orig;
}
String anonymousString = orig;
// ensure non-null
folderDelimiter = StringUtils.defaultString(folderDelimiter);
replacementValue = StringUtils.defaultString(replacementValue);
// replace
for (String valueToReplace : valuesToReplace) {
if (StringUtils.isNotEmpty(valueToReplace)) {
anonymousString = StringUtils.replace(anonymousString,
folderDelimiter + valueToReplace + folderDelimiter,
folderDelimiter + replacementValue + folderDelimiter);
}
}
return anonymousString;
}
/** /**
* Returns true if IP Address is Any Local / Site Local / Link Local / Loop * Returns true if IP Address is Any Local / Site Local / Link Local / Loop
* back local. Sample list "0.0.0.0", wildcard addres * back local. Sample list "0.0.0.0", wildcard addres
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment