From b3bafd8a225916d2110fccf153d2161fa590fbfa Mon Sep 17 00:00:00 2001
From: Greg DiCristofaro <gregd@basistech.com>
Date: Mon, 21 Aug 2023 11:59:47 -0400
Subject: [PATCH] file upload api changes
---
.../cybertriage/autopsy/ctapi/CTApiDAO.java | 9 ++--
.../autopsy/ctapi/CTCloudHttpClient.java | 41 +++++++++++--------
.../autopsy/ctapi/json/AuthTokenRequest.java | 13 ++++++
.../malwarescan/MalwareScanIngestModule.java | 6 ++-
4 files changed, 45 insertions(+), 24 deletions(-)
diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTApiDAO.java b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTApiDAO.java
index c358feec61..e17e771b2a 100644
--- a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTApiDAO.java
+++ b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTApiDAO.java
@@ -78,13 +78,14 @@ public LicenseResponse getLicenseInfo(String licenseString) throws CTCloudExcept
}
public AuthTokenResponse getAuthToken(DecryptedLicenseResponse decrypted) throws CTCloudException {
- return getAuthToken(decrypted, false);
+ return getAuthToken(decrypted, null);
}
- public AuthTokenResponse getAuthToken(DecryptedLicenseResponse decrypted, boolean fileUpload) throws CTCloudException {
+ public AuthTokenResponse getAuthToken(DecryptedLicenseResponse decrypted, Long fileUploadSize) throws CTCloudException {
AuthTokenRequest authTokenRequest = new AuthTokenRequest()
.setAutopsyVersion(getAppVersion())
- .setRequestFileUpload(fileUpload)
+ .setRequestFileUpload(fileUploadSize != null && fileUploadSize > 0)
+ .setFileUploadSize(fileUploadSize != null && fileUploadSize > 0 ? fileUploadSize : null)
.setBoostLicenseId(decrypted.getBoostLicenseId())
.setHostId(decrypted.getLicenseHostId());
@@ -92,7 +93,7 @@ public AuthTokenResponse getAuthToken(DecryptedLicenseResponse decrypted, boolea
}
public void uploadFile(String url, String fileName, InputStream fileIs) throws CTCloudException {
- httpClient.doFileUploadPost(url, fileName, fileIs);
+ httpClient.doFileUploadPut(url, fileName, fileIs);
}
public void uploadMeta(AuthenticatedRequestData authenticatedRequestData, MetadataUploadRequest metaRequest) throws CTCloudException {
diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTCloudHttpClient.java b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTCloudHttpClient.java
index 0a280c87e8..a5a1e0b554 100644
--- a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTCloudHttpClient.java
+++ b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTCloudHttpClient.java
@@ -55,11 +55,13 @@
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpPut;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.util.EntityUtils;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.entity.ContentType;
+import org.apache.http.entity.InputStreamEntity;
import org.apache.http.entity.StringEntity;
import org.apache.http.entity.mime.MultipartEntityBuilder;
import org.sleuthkit.autopsy.coreutils.Logger;
@@ -184,7 +186,7 @@ public <O> O doPost(String urlPath, Map<String, String> urlReqParams, Object jso
return null;
}
- public void doFileUploadPost(String fullUrlPath, String fileName, InputStream fileIs) throws CTCloudException {
+ public void doFileUploadPut(String fullUrlPath, String fileName, InputStream fileIs) throws CTCloudException {
URI postUri;
try {
postUri = new URI(fullUrlPath);
@@ -195,23 +197,26 @@ public void doFileUploadPost(String fullUrlPath, String fileName, InputStream fi
try (CloseableHttpClient httpclient = createConnection(proxySelector, sslContext)) {
LOGGER.log(Level.INFO, "initiating http post request to ctcloud server " + fullUrlPath);
- HttpPost post = new HttpPost(postUri);
- configureRequestTimeout(post);
-
- post.addHeader("Connection", "keep-alive");
-
- MultipartEntityBuilder builder = MultipartEntityBuilder.create();
- builder.addBinaryBody(
- "file",
- fileIs,
- ContentType.APPLICATION_OCTET_STREAM,
- fileName
- );
-
- HttpEntity multipart = builder.build();
- post.setEntity(multipart);
-
- try (CloseableHttpResponse response = httpclient.execute(post)) {
+ HttpPut put = new HttpPut(postUri);
+ configureRequestTimeout(put);
+
+ put.addHeader("Connection", "keep-alive");
+ put.addHeader("Content-Type", "application/octet-stream");
+
+// MultipartEntityBuilder builder = MultipartEntityBuilder.create();
+// builder.addBinaryBody(
+// "file",
+// fileBytes,
+// ContentType.APPLICATION_OCTET_STREAM,
+// file.getFileName()
+// );
+//
+// HttpEntity multipart = builder.build();
+// post.setEntity(multipart);
+
+ put.setEntity(new InputStreamEntity(fileIs));
+
+ try (CloseableHttpResponse response = httpclient.execute(put)) {
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode == HttpStatus.SC_OK || statusCode == HttpStatus.SC_NO_CONTENT) {
LOGGER.log(Level.INFO, "Response Received. - Status OK");
diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/json/AuthTokenRequest.java b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/json/AuthTokenRequest.java
index 6a093fee8e..f137818346 100644
--- a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/json/AuthTokenRequest.java
+++ b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/json/AuthTokenRequest.java
@@ -34,6 +34,9 @@ public class AuthTokenRequest {
@JsonProperty("requestFileUpload")
private boolean requestFileUpload;
+ @JsonProperty("fileUploadSize")
+ private Long fileUploadSize;
+
@JsonProperty("host_id")
private String hostId;
@@ -64,6 +67,16 @@ public AuthTokenRequest setRequestFileUpload(boolean requestFileUpload) {
return this;
}
+ public Long getFileUploadSize() {
+ return fileUploadSize;
+ }
+
+ public AuthTokenRequest setFileUploadSize(Long fileUploadSize) {
+ this.fileUploadSize = fileUploadSize;
+ return this;
+ }
+
+
public String getHostId() {
return hostId;
}
diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java b/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java
index 4698392478..cbe76ba235 100644
--- a/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java
+++ b/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java
@@ -105,8 +105,10 @@ private static class SharedProcessing {
//minimum file uploads left before issuing warning
private static final long LOW_UPLOADS_REMAINING = 25;
+ // min and max upload size in bytes
private static final long MIN_UPLOAD_SIZE = 1;
- private static final long MAX_UPLOAD_SIZE = 1_000_000_000;
+ private static final long MAX_UPLOAD_SIZE = 100_000_000; // 100MB
+
private static final int NUM_FILE_UPLOAD_RETRIES = 7;
private static final long FILE_UPLOAD_RETRY_SLEEP_MILLIS = 60 * 1000;
@@ -634,7 +636,7 @@ private boolean uploadFile(IngestJobState ingestJobState, String md5, long objId
}
// get auth token / file upload url
- AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(ingestJobState.getLicenseInfo().getDecryptedLicense(), true);
+ AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(ingestJobState.getLicenseInfo().getDecryptedLicense(), af.getSize());
if (StringUtils.isBlank(authTokenResponse.getFileUploadUrl())) {
throw new CTCloudException(CTCloudException.ErrorCode.NETWORK_ERROR);
} else if (remaining(authTokenResponse.getFileUploadLimit(), authTokenResponse.getFileUploadCount()) <= 0) {
--
GitLab