diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTCloudHttpClient.java b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTCloudHttpClient.java
index 91a180e9bc787b6812b109652ba1e19a87929481..06e5fb7fcd3cca27be8700b73864020cef16a51c 100644
--- a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTCloudHttpClient.java
+++ b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTCloudHttpClient.java
@@ -200,6 +200,7 @@ public <O> O doPost(String urlPath, Map<String, String> urlReqParams, Object jso
public void doFileUploadPost(String urlPath, String fileName, InputStream fileIs) throws CTCloudException {
try (CloseableHttpClient httpclient = createConnection(getProxySettings(), sslContext)) {
+ LOGGER.log(Level.INFO, "initiating http post request to ctcloud server " + urlPath);
HttpPost post = new HttpPost(urlPath);
configureRequestTimeout(post);
diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java b/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java
index 56bfde735610b41a63fdc73e3b9d579a2ec39dfd..5cbbbed372a69a11d109b90d53e15c3b295ebeab 100644
--- a/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java
+++ b/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java
@@ -27,10 +27,14 @@
import com.basistech.df.cybertriage.autopsy.ctapi.json.MalwareResultBean.Status;
import com.basistech.df.cybertriage.autopsy.ctapi.json.MetadataUploadRequest;
import com.basistech.df.cybertriage.autopsy.ctoptions.ctcloud.CTLicensePersistence;
+import java.security.DigestInputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HexFormat;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -133,6 +137,7 @@ private static class SharedProcessing {
private final CTLicensePersistence ctSettingsPersistence = CTLicensePersistence.getInstance();
private final CTApiDAO ctApiDAO = CTApiDAO.getInstance();
+ private final UsernameAnonymizer usernameAnonymizer = new UsernameAnonymizer();
private IngestJobState ingestJobState = null;
@@ -258,31 +263,86 @@ private static long remaining(Long limit, Long used) {
* @param af The abstract file.
* @return The md5 hash (or null if could not be determined).
*/
- private static String getOrCalcHash(AbstractFile af) {
- if (StringUtils.isNotBlank(af.getMd5Hash())) {
- return af.getMd5Hash();
+ private static String getOrCalcHash(AbstractFile af, HashType hashType) {
+ switch (hashType) {
+ case MD5:
+ if (StringUtils.isNotBlank(af.getMd5Hash())) {
+ return af.getMd5Hash();
+ }
+ break;
+ case SHA256:
+ if (StringUtils.isNotBlank(af.getSha256Hash())) {
+ return af.getSha256Hash();
+ }
}
try {
- List<HashResult> hashResults = HashUtility.calculateHashes(af, Collections.singletonList(HashType.MD5));
+ List<HashResult> hashResults = HashUtility.calculateHashes(af, Collections.singletonList(hashType));
if (CollectionUtils.isNotEmpty(hashResults)) {
for (HashResult hashResult : hashResults) {
- if (hashResult.getType() == HashType.MD5) {
+ if (hashResult.getType() == hashType) {
return hashResult.getValue();
}
}
}
} catch (TskCoreException ex) {
logger.log(Level.WARNING,
- MessageFormat.format("An error occurred while processing file name: {0} and obj id: {1}.",
+ MessageFormat.format("An error occurred while processing hash for file name: {0} and obj id: {1} and hash type {2}.",
af.getName(),
- af.getId()),
+ af.getId(),
+ hashType.name()),
ex);
}
return null;
}
+ /**
+ * Gets or calculates the md5 for a file.
+ *
+ * @param af The file.
+ * @return The hash.
+ */
+ private static String getOrCalcMd5(AbstractFile af) {
+ return getOrCalcHash(af, HashType.MD5);
+ }
+
+ /**
+ * Gets or calculates the sha256 for a file.
+ *
+ * @param af The file.
+ * @return The hash.
+ */
+ private static String getOrCalcSha256(AbstractFile af) {
+ return getOrCalcHash(af, HashType.SHA256);
+ }
+
+ /**
+ * Gets or calculates the sha1 for a file.
+ *
+ * @param af The file.
+ * @return The hash.
+ */
+ private static String getOrCalcSha1(AbstractFile af) throws NoSuchAlgorithmException, ReadContentInputStream.ReadContentInputStreamException {
+ if (StringUtils.isNotBlank(af.getSha1Hash())) {
+ return af.getSha1Hash();
+ }
+ // taken from https://stackoverflow.com/questions/6293713/java-how-to-create-sha-1-for-a-file
+ MessageDigest digest = MessageDigest.getInstance("SHA-1");
+ ReadContentInputStream afStream = new ReadContentInputStream(af);
+ int n = 0;
+ byte[] buffer = new byte[8192];
+ while (n != -1) {
+ n = afStream.read(buffer);
+ if (n > 0) {
+ digest.update(buffer, 0, n);
+ }
+ }
+ byte[] hashBytes = digest.digest();
+ String hashString = HexFormat.of().formatHex(hashBytes);
+ return hashString;
+ }
+
/**
* Processes a file. The file goes through the lookup process if the
* file meets acceptable criteria: 1) not FileKnown.KNOWN 2) is
@@ -305,7 +365,7 @@ IngestModule.ProcessResult process(AbstractFile af) {
&& EXECUTABLE_MIME_TYPES.contains(StringUtils.defaultString(ingestJobState.getFileTypeDetector().getMIMEType(af)).trim().toLowerCase())
&& CollectionUtils.isEmpty(af.getAnalysisResults(ingestJobState.getMalwareType()))) {
- String md5 = getOrCalcHash(af);
+ String md5 = getOrCalcMd5(af);
if (StringUtils.isNotBlank(md5)) {
batchProcessor.add(new FileRecord(af.getId(), md5));
}
@@ -393,12 +453,11 @@ private void handleBatch(IngestJobState ingestJobState, List<FileRecord> fileRec
* @param repResult The ct cloud results.
* @throws org.sleuthkit.datamodel.Blackboard.BlackboardException
* @throws TskCoreException
- * @throws TskCoreException
*/
@Messages({
"MalwareScanIngestModule_SharedProcessing_exhaustedResultsHashLookups_title=Some Lookup Results Not Processed",
"MalwareScanIngestModule_SharedProcessing_exhaustedResultsHashLookups_desc=Some lookup results were not processed due to exceeding limits. Please try again later.",})
- private void handleLookupResults(IngestJobState ingestJobState, Map<String, List<Long>> md5ToObjId, List<CTCloudBean> repResult) throws Blackboard.BlackboardException, TskCoreException, TskCoreException, CTCloudException {
+ private void handleLookupResults(IngestJobState ingestJobState, Map<String, List<Long>> md5ToObjId, List<CTCloudBean> repResult) throws Blackboard.BlackboardException, TskCoreException, TskCoreException, CTCloudException, NoSuchAlgorithmException, ReadContentInputStream.ReadContentInputStreamException {
if (CollectionUtils.isEmpty(repResult)) {
return;
}
@@ -445,7 +504,7 @@ private void handleLookupResults(IngestJobState ingestJobState, Map<String, List
* @param performFileUpload True if the class of results warrants file
* upload (i.e. NOT_FOUND)
*/
- private void handleNonFoundResults(IngestJobState ingestJobState, Map<String, List<Long>> md5ToObjId, List<CTCloudBean> results, boolean performFileUpload) throws CTCloudException, TskCoreException {
+ private void handleNonFoundResults(IngestJobState ingestJobState, Map<String, List<Long>> md5ToObjId, List<CTCloudBean> results, boolean performFileUpload) throws CTCloudException, TskCoreException, NoSuchAlgorithmException, ReadContentInputStream.ReadContentInputStreamException {
if (CollectionUtils.isNotEmpty(results)
&& ingestJobState.isDoFileLookups()
&& ((performFileUpload && ingestJobState.isUploadUnknownFiles()) || (!performFileUpload && ingestJobState.isQueryForMissing()))) {
@@ -539,7 +598,7 @@ private static boolean isUploadable(AbstractFile af) {
* @throws CTCloudException
* @throws TskCoreException
*/
- private boolean uploadFile(IngestJobState ingestJobState, String md5, long objId) throws CTCloudException, TskCoreException {
+ private boolean uploadFile(IngestJobState ingestJobState, String md5, long objId) throws CTCloudException, TskCoreException, NoSuchAlgorithmException, ReadContentInputStream.ReadContentInputStreamException {
if (!ingestJobState.isUploadUnknownFiles() || ingestJobState.getIngestJobContext().fileIngestIsCancelled()) {
return false;
}
@@ -568,13 +627,12 @@ private boolean uploadFile(IngestJobState ingestJobState, String md5, long objId
// upload metadata
MetadataUploadRequest metaRequest = new MetadataUploadRequest()
.setCreatedDate(af.getCrtime() == 0 ? null : af.getCrtime())
- .setFilePath(af.getUniquePath())
+ .setFilePath(usernameAnonymizer.anonymousUsername(af.getUniquePath()))
.setFileSizeBytes(af.getSize())
.setFileUploadUrl(authTokenResponse.getFileUploadUrl())
.setMd5(md5)
- // these may be missing, but that's fine
- .setSha1(af.getSha1Hash())
- .setSha256(af.getSha256Hash());
+ .setSha1(getOrCalcSha1(af))
+ .setSha256(getOrCalcSha256(af));
ctApiDAO.uploadMeta(new AuthenticatedRequestData(ingestJobState.getLicenseInfo().getDecryptedLicense(), authTokenResponse), metaRequest);
return true;
@@ -627,7 +685,7 @@ private void longPollForNotFound(IngestJobState ingestJobState) throws Interrupt
createAnalysisResults(ingestJobState, found, remaining);
// remove any found items from the list of items to long poll for
- for (CTCloudBean foundItem : found) {
+ for (CTCloudBean foundItem : CollectionUtils.emptyIfNull(found)) {
String normalizedMd5 = normalizedMd5(foundItem.getMd5HashValue());
remaining.remove(normalizedMd5);
}
@@ -768,8 +826,8 @@ synchronized void shutDown() {
// flush any remaining items
try {
- longPollForNotFound(ingestJobState);
batchProcessor.flushAndReset();
+ longPollForNotFound(ingestJobState);
} catch (InterruptedException ex) {
notifyWarning(
Bundle.MalwareScanIngestModule_SharedProcessing_flushTimeout_title(),