diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java b/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java
index e249b487d5d5972749f4979712666315f1cf9e39..2a67b4679221aa8d864d5d152657cd242f47eb6b 100644
--- a/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java
+++ b/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java
@@ -26,6 +26,7 @@
import com.basistech.df.cybertriage.autopsy.ctapi.json.DecryptedLicenseResponse;
import com.basistech.df.cybertriage.autopsy.ctapi.json.LicenseInfo;
import com.basistech.df.cybertriage.autopsy.ctapi.json.MalwareResultBean;
+import com.basistech.df.cybertriage.autopsy.ctapi.json.MalwareResultBean.Status;
import com.basistech.df.cybertriage.autopsy.ctapi.json.MetadataUploadRequest;
import com.basistech.df.cybertriage.autopsy.ctoptions.ctcloud.CTLicensePersistence;
import java.text.MessageFormat;
@@ -103,7 +104,6 @@ private static class SharedProcessing {
private static final long MAX_UPLOAD_SIZE = 1_000_000_000;
private static final int NUM_FILE_UPLOAD_RETRIES = 60 * 5;
private static final long FILE_UPLOAD_RETRY_SLEEP_MILLIS = 60 * 1000;
-
private static final Set<String> EXECUTABLE_MIME_TYPES = Stream.of(
"application/x-bat",//NON-NLS
@@ -128,6 +128,7 @@ private static class SharedProcessing {
private final CTLicensePersistence ctSettingsPersistence = CTLicensePersistence.getInstance();
private final CTApiDAO ctApiDAO = CTApiDAO.getInstance();
+ // TODO minimize state
private RunState runState = null;
private SleuthkitCase tskCase = null;
@@ -137,6 +138,7 @@ private static class SharedProcessing {
private long dsId = 0;
private long ingestJobId = 0;
private boolean uploadUnknownFiles = false;
+ private Map<String, List<Long>> unidentifiedHashes = null;
@Messages({
"MalwareScanIngestModule_ShareProcessing_lowLimitWarning_title=Hash Lookups Low",
@@ -196,7 +198,8 @@ synchronized void startUp(IngestJobContext context) throws IngestModuleException
ingestJobId = context.getJobId();
licenseInfo = licenseInfoOpt.get();
uploadUnknownFiles = ctSettingsPersistence.loadMalwareIngestSettings().isUploadFiles();
-
+ unidentifiedHashes = new HashMap<>();
+
// set run state to initialized
runState = RunState.STARTED_UP;
} catch (Exception ex) {
@@ -306,66 +309,79 @@ private void handleBatch(List<FileRecord> fileRecords) {
}
try {
- // get an auth token with the license
- AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(licenseInfo.getDecryptedLicense());
-
- // make sure we are in bounds for the remaining scans
- long remainingScans = remaining(authTokenResponse.getHashLookupLimit(), authTokenResponse.getHashLookupCount());
- if (remainingScans <= 0) {
- runState = RunState.DISABLED;
- notifyWarning(
- Bundle.MalwareScanIngestModule_SharedProcessing_exhaustedHashLookups_title(),
- Bundle.MalwareScanIngestModule_SharedProcessing_exhaustedHashLookups_desc(),
- null);
- return;
- }
+ List<CTCloudBean> repResult = getHashLookupResults(md5Hashes);
+ Map<Boolean, List<CTCloudBean>> partitioned = repResult.stream()
+ .filter(bean -> bean.getMalwareResult() != null)
+ .collect(Collectors.partitioningBy(bean -> bean.getMalwareResult().getStatus() == Status.FOUND));
+
+ // TODO handle caching list and creating new items
+
+ createArtifacts(repResult, md5ToObjId);
+ } catch (Exception ex) {
+ notifyWarning(
+ Bundle.MalwareScanIngestModule_SharedProcessing_generalProcessingError_title(),
+ Bundle.MalwareScanIngestModule_SharedProcessing_generalProcessingError_desc(),
+ ex);
+ }
+ }
- // using auth token, get results
- List<CTCloudBean> repResult = ctApiDAO.getReputationResults(
- new AuthenticatedRequestData(licenseInfo.getDecryptedLicense(), authTokenResponse),
- md5Hashes
- );
-
- List<BlackboardArtifact> createdArtifacts = new ArrayList<>();
- if (!CollectionUtils.isEmpty(repResult)) {
- SleuthkitCase.CaseDbTransaction trans = null;
- try {
- trans = tskCase.beginTransaction();
- for (CTCloudBean result : repResult) {
- String sanitizedMd5 = sanitizedMd5(result.getMd5HashValue());
- List<Long> objIds = md5ToObjId.remove(sanitizedMd5);
- if (objIds == null || objIds.isEmpty()) {
- continue;
- }
+ private void createArtifacts(List<CTCloudBean> repResult, Map<String, List<Long>> md5ToObjId) throws Blackboard.BlackboardException, TskCoreException {
+ List<BlackboardArtifact> createdArtifacts = new ArrayList<>();
+ if (!CollectionUtils.isEmpty(repResult)) {
+ SleuthkitCase.CaseDbTransaction trans = null;
+ try {
+ trans = tskCase.beginTransaction();
+ for (CTCloudBean result : repResult) {
+ String sanitizedMd5 = sanitizedMd5(result.getMd5HashValue());
+ List<Long> objIds = md5ToObjId.remove(sanitizedMd5);
+ if (objIds == null || objIds.isEmpty()) {
+ continue;
+ }
- for (Long objId : objIds) {
- AnalysisResult res = createAnalysisResult(objId, result, trans);
- if (res != null) {
- createdArtifacts.add(res);
- }
+ for (Long objId : objIds) {
+ AnalysisResult res = createAnalysisResult(objId, result, trans);
+ if (res != null) {
+ createdArtifacts.add(res);
}
}
+ }
- trans.commit();
+ trans.commit();
+ trans = null;
+ } finally {
+ if (trans != null) {
+ trans.rollback();
+ createdArtifacts.clear();
trans = null;
- } finally {
- if (trans != null) {
- trans.rollback();
- createdArtifacts.clear();
- trans = null;
- }
}
+ }
- if (!CollectionUtils.isEmpty(createdArtifacts)) {
- tskCase.getBlackboard().postArtifacts(createdArtifacts, Bundle.MalwareScanIngestModuleFactory_displayName(), ingestJobId);
- }
+ if (!CollectionUtils.isEmpty(createdArtifacts)) {
+ tskCase.getBlackboard().postArtifacts(createdArtifacts, Bundle.MalwareScanIngestModuleFactory_displayName(), ingestJobId);
}
- } catch (Exception ex) {
+ }
+ }
+
+ private List<CTCloudBean> getHashLookupResults(List<String> md5Hashes) throws CTCloudException {
+ // get an auth token with the license
+ AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(licenseInfo.getDecryptedLicense());
+
+ // make sure we are in bounds for the remaining scans
+ long remainingScans = remaining(authTokenResponse.getHashLookupLimit(), authTokenResponse.getHashLookupCount());
+ if (remainingScans <= 0) {
+ runState = RunState.DISABLED;
notifyWarning(
- Bundle.MalwareScanIngestModule_SharedProcessing_generalProcessingError_title(),
- Bundle.MalwareScanIngestModule_SharedProcessing_generalProcessingError_desc(),
- ex);
+ Bundle.MalwareScanIngestModule_SharedProcessing_exhaustedHashLookups_title(),
+ Bundle.MalwareScanIngestModule_SharedProcessing_exhaustedHashLookups_desc(),
+ null);
+ return Collections.emptyList();
}
+
+ // using auth token, get results
+ return ctApiDAO.getReputationResults(
+ new AuthenticatedRequestData(licenseInfo.getDecryptedLicense(), authTokenResponse),
+ md5Hashes
+ );
}
private String sanitizedMd5(String orig) {
@@ -392,7 +408,7 @@ private boolean uploadFile(CTCloudBean cloudBean, long objId) throws CTCloudExce
return false;
}
- AbstractFile af = skCase.getAbstractFileById(objId);
+ AbstractFile af = tskCase.getAbstractFileById(objId);
if (af == null) {
return false;
}
@@ -402,7 +418,7 @@ private boolean uploadFile(CTCloudBean cloudBean, long objId) throws CTCloudExce
}
// get auth token / file upload url
- AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(decrypted, true);
+ AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(licenseInfo.getDecryptedLicense(), true);
if (StringUtils.isBlank(authTokenResponse.getFileUploadUrl())) {
throw new CTCloudException(CTCloudException.ErrorCode.NETWORK_ERROR);
} else if (remaining(authTokenResponse.getFileUploadLimit(), authTokenResponse.getFileUploadCount()) <= 0) {
@@ -425,27 +441,29 @@ private boolean uploadFile(CTCloudBean cloudBean, long objId) throws CTCloudExce
.setSha1(af.getSha1Hash())
.setSha256(af.getSha256Hash());
- ctApiDAO.uploadMeta(new AuthenticatedRequestData(decrypted, authTokenResponse), metaRequest);
+ ctApiDAO.uploadMeta(new AuthenticatedRequestData(licenseInfo.getDecryptedLicense(), authTokenResponse), metaRequest);
return true;
}
- private boolean getUploadedFileResults(Map<String, List<Long>> md5objIdMapping) {
+ private boolean getUploadedFileResults(Map<String, List<Long>> md5objIdMapping) throws InterruptedException, CTCloudException, Blackboard.BlackboardException, TskCoreException {
+ // TODO integrate this
Map<String, List<Long>> remaining = new HashMap<>(md5objIdMapping);
for (int retry = 0; retry < NUM_FILE_UPLOAD_RETRIES; retry++) {
List<List<String>> md5Batches = Lists.partition(new ArrayList<>(remaining.keySet()), BATCH_SIZE);
for (List<String> batch : md5Batches) {
- // TODO query and capture still unknown
+ List<CTCloudBean> repResult = getHashLookupResults(batch);
+ createArtifacts(repResult, remaining);
}
-
+
if (remaining.isEmpty()) {
return true;
}
-
-
+
Thread.sleep(FILE_UPLOAD_RETRY_SLEEP_MILLIS);
}
+ return false;
}
@Messages({