Skip to content
Snippets Groups Projects
Commit 032993c8 authored by Greg DiCristofaro's avatar Greg DiCristofaro
Browse files

updates

parent f23331b6
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java
+ 77
− 59
View file @ 032993c8
......@@ -26,6 +26,7 @@
import com.basistech.df.cybertriage.autopsy.ctapi.json.DecryptedLicenseResponse;
import com.basistech.df.cybertriage.autopsy.ctapi.json.LicenseInfo;
import com.basistech.df.cybertriage.autopsy.ctapi.json.MalwareResultBean;
import com.basistech.df.cybertriage.autopsy.ctapi.json.MalwareResultBean.Status;
import com.basistech.df.cybertriage.autopsy.ctapi.json.MetadataUploadRequest;
import com.basistech.df.cybertriage.autopsy.ctoptions.ctcloud.CTLicensePersistence;
import java.text.MessageFormat;
......@@ -103,7 +104,6 @@ private static class SharedProcessing {
private static final long MAX_UPLOAD_SIZE = 1_000_000_000;
private static final int NUM_FILE_UPLOAD_RETRIES = 60 * 5;
private static final long FILE_UPLOAD_RETRY_SLEEP_MILLIS = 60 * 1000;
private static final Set<String> EXECUTABLE_MIME_TYPES = Stream.of(
"application/x-bat",//NON-NLS
......@@ -128,6 +128,7 @@ private static class SharedProcessing {
private final CTLicensePersistence ctSettingsPersistence = CTLicensePersistence.getInstance();
private final CTApiDAO ctApiDAO = CTApiDAO.getInstance();
// TODO minimize state
private RunState runState = null;
private SleuthkitCase tskCase = null;
......@@ -137,6 +138,7 @@ private static class SharedProcessing {
private long dsId = 0;
private long ingestJobId = 0;
private boolean uploadUnknownFiles = false;
private Map<String, List<Long>> unidentifiedHashes = null;
@Messages({
"MalwareScanIngestModule_ShareProcessing_lowLimitWarning_title=Hash Lookups Low",
......@@ -196,7 +198,8 @@ synchronized void startUp(IngestJobContext context) throws IngestModuleException
ingestJobId = context.getJobId();
licenseInfo = licenseInfoOpt.get();
uploadUnknownFiles = ctSettingsPersistence.loadMalwareIngestSettings().isUploadFiles();
unidentifiedHashes = new HashMap<>();
// set run state to initialized
runState = RunState.STARTED_UP;
} catch (Exception ex) {
......@@ -306,66 +309,79 @@ private void handleBatch(List<FileRecord> fileRecords) {
}
try {
// get an auth token with the license
AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(licenseInfo.getDecryptedLicense());
// make sure we are in bounds for the remaining scans
long remainingScans = remaining(authTokenResponse.getHashLookupLimit(), authTokenResponse.getHashLookupCount());
if (remainingScans <= 0) {
runState = RunState.DISABLED;
notifyWarning(
Bundle.MalwareScanIngestModule_SharedProcessing_exhaustedHashLookups_title(),
Bundle.MalwareScanIngestModule_SharedProcessing_exhaustedHashLookups_desc(),
null);
return;
}
List<CTCloudBean> repResult = getHashLookupResults(md5Hashes);
Map<Boolean, List<CTCloudBean>> partitioned = repResult.stream()
.filter(bean -> bean.getMalwareResult() != null)
.collect(Collectors.partitioningBy(bean -> bean.getMalwareResult().getStatus() == Status.FOUND));
// TODO handle caching list and creating new items
createArtifacts(repResult, md5ToObjId);
} catch (Exception ex) {
notifyWarning(
Bundle.MalwareScanIngestModule_SharedProcessing_generalProcessingError_title(),
Bundle.MalwareScanIngestModule_SharedProcessing_generalProcessingError_desc(),
ex);
}
}
// using auth token, get results
List<CTCloudBean> repResult = ctApiDAO.getReputationResults(
new AuthenticatedRequestData(licenseInfo.getDecryptedLicense(), authTokenResponse),
md5Hashes
);
List<BlackboardArtifact> createdArtifacts = new ArrayList<>();
if (!CollectionUtils.isEmpty(repResult)) {
SleuthkitCase.CaseDbTransaction trans = null;
try {
trans = tskCase.beginTransaction();
for (CTCloudBean result : repResult) {
String sanitizedMd5 = sanitizedMd5(result.getMd5HashValue());
List<Long> objIds = md5ToObjId.remove(sanitizedMd5);
if (objIds == null || objIds.isEmpty()) {
continue;
}
private void createArtifacts(List<CTCloudBean> repResult, Map<String, List<Long>> md5ToObjId) throws Blackboard.BlackboardException, TskCoreException {
List<BlackboardArtifact> createdArtifacts = new ArrayList<>();
if (!CollectionUtils.isEmpty(repResult)) {
SleuthkitCase.CaseDbTransaction trans = null;
try {
trans = tskCase.beginTransaction();
for (CTCloudBean result : repResult) {
String sanitizedMd5 = sanitizedMd5(result.getMd5HashValue());
List<Long> objIds = md5ToObjId.remove(sanitizedMd5);
if (objIds == null || objIds.isEmpty()) {
continue;
}
for (Long objId : objIds) {
AnalysisResult res = createAnalysisResult(objId, result, trans);
if (res != null) {
createdArtifacts.add(res);
}
for (Long objId : objIds) {
AnalysisResult res = createAnalysisResult(objId, result, trans);
if (res != null) {
createdArtifacts.add(res);
}
}
}
trans.commit();
trans.commit();
trans = null;
} finally {
if (trans != null) {
trans.rollback();
createdArtifacts.clear();
trans = null;
} finally {
if (trans != null) {
trans.rollback();
createdArtifacts.clear();
trans = null;
}
}
}
if (!CollectionUtils.isEmpty(createdArtifacts)) {
tskCase.getBlackboard().postArtifacts(createdArtifacts, Bundle.MalwareScanIngestModuleFactory_displayName(), ingestJobId);
}
if (!CollectionUtils.isEmpty(createdArtifacts)) {
tskCase.getBlackboard().postArtifacts(createdArtifacts, Bundle.MalwareScanIngestModuleFactory_displayName(), ingestJobId);
}
} catch (Exception ex) {
}
}
private List<CTCloudBean> getHashLookupResults(List<String> md5Hashes) throws CTCloudException {
// get an auth token with the license
AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(licenseInfo.getDecryptedLicense());
// make sure we are in bounds for the remaining scans
long remainingScans = remaining(authTokenResponse.getHashLookupLimit(), authTokenResponse.getHashLookupCount());
if (remainingScans <= 0) {
runState = RunState.DISABLED;
notifyWarning(
Bundle.MalwareScanIngestModule_SharedProcessing_generalProcessingError_title(),
Bundle.MalwareScanIngestModule_SharedProcessing_generalProcessingError_desc(),
ex);
Bundle.MalwareScanIngestModule_SharedProcessing_exhaustedHashLookups_title(),
Bundle.MalwareScanIngestModule_SharedProcessing_exhaustedHashLookups_desc(),
null);
return Collections.emptyList();
}
// using auth token, get results
return ctApiDAO.getReputationResults(
new AuthenticatedRequestData(licenseInfo.getDecryptedLicense(), authTokenResponse),
md5Hashes
);
}
private String sanitizedMd5(String orig) {
......@@ -392,7 +408,7 @@ private boolean uploadFile(CTCloudBean cloudBean, long objId) throws CTCloudExce
return false;
}
AbstractFile af = skCase.getAbstractFileById(objId);
AbstractFile af = tskCase.getAbstractFileById(objId);
if (af == null) {
return false;
}
......@@ -402,7 +418,7 @@ private boolean uploadFile(CTCloudBean cloudBean, long objId) throws CTCloudExce
}
// get auth token / file upload url
AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(decrypted, true);
AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(licenseInfo.getDecryptedLicense(), true);
if (StringUtils.isBlank(authTokenResponse.getFileUploadUrl())) {
throw new CTCloudException(CTCloudException.ErrorCode.NETWORK_ERROR);
} else if (remaining(authTokenResponse.getFileUploadLimit(), authTokenResponse.getFileUploadCount()) <= 0) {
......@@ -425,27 +441,29 @@ private boolean uploadFile(CTCloudBean cloudBean, long objId) throws CTCloudExce
.setSha1(af.getSha1Hash())
.setSha256(af.getSha256Hash());
ctApiDAO.uploadMeta(new AuthenticatedRequestData(decrypted, authTokenResponse), metaRequest);
ctApiDAO.uploadMeta(new AuthenticatedRequestData(licenseInfo.getDecryptedLicense(), authTokenResponse), metaRequest);
return true;
}
private boolean getUploadedFileResults(Map<String, List<Long>> md5objIdMapping) {
private boolean getUploadedFileResults(Map<String, List<Long>> md5objIdMapping) throws InterruptedException, CTCloudException, Blackboard.BlackboardException, TskCoreException {
// TODO integrate this
Map<String, List<Long>> remaining = new HashMap<>(md5objIdMapping);
for (int retry = 0; retry < NUM_FILE_UPLOAD_RETRIES; retry++) {
List<List<String>> md5Batches = Lists.partition(new ArrayList<>(remaining.keySet()), BATCH_SIZE);
for (List<String> batch : md5Batches) {
// TODO query and capture still unknown
List<CTCloudBean> repResult = getHashLookupResults(batch);
createArtifacts(repResult, remaining);
}
if (remaining.isEmpty()) {
return true;
}
Thread.sleep(FILE_UPLOAD_RETRY_SLEEP_MILLIS);
}
return false;
}
@Messages({
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment