diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTApiDAO.java b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTApiDAO.java
index c358feec61a460b49c71addeeb741237198a26f6..f21873f565755fc9754694d6faa6a2b8327fd029 100644
--- a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTApiDAO.java
+++ b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTApiDAO.java
@@ -25,6 +25,7 @@
import com.basistech.df.cybertriage.autopsy.ctapi.json.CTCloudBeanResponse;
import com.basistech.df.cybertriage.autopsy.ctapi.json.DecryptedLicenseResponse;
import com.basistech.df.cybertriage.autopsy.ctapi.json.FileReputationRequest;
+import com.basistech.df.cybertriage.autopsy.ctapi.json.FileUploadRequest;
import com.basistech.df.cybertriage.autopsy.ctapi.json.LicenseRequest;
import com.basistech.df.cybertriage.autopsy.ctapi.json.LicenseResponse;
import com.basistech.df.cybertriage.autopsy.ctapi.json.MetadataUploadRequest;
@@ -78,21 +79,22 @@ public LicenseResponse getLicenseInfo(String licenseString) throws CTCloudExcept
}
public AuthTokenResponse getAuthToken(DecryptedLicenseResponse decrypted) throws CTCloudException {
- return getAuthToken(decrypted, false);
+ return getAuthToken(decrypted, null);
}
- public AuthTokenResponse getAuthToken(DecryptedLicenseResponse decrypted, boolean fileUpload) throws CTCloudException {
+ public AuthTokenResponse getAuthToken(DecryptedLicenseResponse decrypted, Long fileUploadSize) throws CTCloudException {
AuthTokenRequest authTokenRequest = new AuthTokenRequest()
.setAutopsyVersion(getAppVersion())
- .setRequestFileUpload(fileUpload)
+ .setRequestFileUpload(fileUploadSize != null && fileUploadSize > 0)
+ .setFileUploadSize(fileUploadSize != null && fileUploadSize > 0 ? fileUploadSize : null)
.setBoostLicenseId(decrypted.getBoostLicenseId())
.setHostId(decrypted.getLicenseHostId());
return httpClient.doPost(AUTH_TOKEN_REQUEST_PATH, authTokenRequest, AuthTokenResponse.class);
}
- public void uploadFile(String url, String fileName, InputStream fileIs) throws CTCloudException {
- httpClient.doFileUploadPost(url, fileName, fileIs);
+ public void uploadFile(FileUploadRequest fileUploadRequest) throws CTCloudException {
+ httpClient.doFileUploadPut(fileUploadRequest);
}
public void uploadMeta(AuthenticatedRequestData authenticatedRequestData, MetadataUploadRequest metaRequest) throws CTCloudException {
diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTCloudHttpClient.java b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTCloudHttpClient.java
index 0a280c87e89bc5f11b52e9f42930620bddf33e2b..0a7c69cdeecb263bffda33a6b227952914bbd518 100644
--- a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTCloudHttpClient.java
+++ b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTCloudHttpClient.java
@@ -18,6 +18,8 @@
*/
package com.basistech.df.cybertriage.autopsy.ctapi;
+import com.basistech.df.cybertriage.autopsy.ctapi.CTCloudException.ErrorCode;
+import com.basistech.df.cybertriage.autopsy.ctapi.json.FileUploadRequest;
import com.basistech.df.cybertriage.autopsy.ctapi.util.ObjectMapperUtil;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.IOException;
@@ -55,13 +57,14 @@
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpPut;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.util.EntityUtils;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.entity.ContentType;
+import org.apache.http.entity.InputStreamEntity;
import org.apache.http.entity.StringEntity;
-import org.apache.http.entity.mime.MultipartEntityBuilder;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
@@ -184,10 +187,23 @@ public <O> O doPost(String urlPath, Map<String, String> urlReqParams, Object jso
return null;
}
- public void doFileUploadPost(String fullUrlPath, String fileName, InputStream fileIs) throws CTCloudException {
- URI postUri;
+ public void doFileUploadPut(FileUploadRequest fileUploadRequest) throws CTCloudException {
+ if (fileUploadRequest == null) {
+ throw new CTCloudException(ErrorCode.BAD_REQUEST, new IllegalArgumentException("fileUploadRequest cannot be null"));
+ }
+
+ String fullUrlPath = fileUploadRequest.getFullUrlPath();
+ String fileName = fileUploadRequest.getFileName();
+ InputStream fileInputStream = fileUploadRequest.getFileInputStream();
+ Long contentLength = fileUploadRequest.getContentLength();
+
+ if (StringUtils.isBlank(fullUrlPath) || fileInputStream == null || contentLength == null || contentLength <= 0) {
+ throw new CTCloudException(ErrorCode.BAD_REQUEST, new IllegalArgumentException("fullUrlPath, fileInputStream, contentLength must not be empty, null or less than 0"));
+ }
+
+ URI putUri;
try {
- postUri = new URI(fullUrlPath);
+ putUri = new URI(fullUrlPath);
} catch (URISyntaxException ex) {
LOGGER.log(Level.WARNING, "Wrong URL syntax for CT Cloud " + fullUrlPath, ex);
throw new CTCloudException(CTCloudException.ErrorCode.UNKNOWN, ex);
@@ -195,23 +211,13 @@ public void doFileUploadPost(String fullUrlPath, String fileName, InputStream fi
try (CloseableHttpClient httpclient = createConnection(proxySelector, sslContext)) {
LOGGER.log(Level.INFO, "initiating http post request to ctcloud server " + fullUrlPath);
- HttpPost post = new HttpPost(postUri);
- configureRequestTimeout(post);
-
- post.addHeader("Connection", "keep-alive");
-
- MultipartEntityBuilder builder = MultipartEntityBuilder.create();
- builder.addBinaryBody(
- "file",
- fileIs,
- ContentType.APPLICATION_OCTET_STREAM,
- fileName
- );
+ HttpPut put = new HttpPut(putUri);
+ configureRequestTimeout(put);
- HttpEntity multipart = builder.build();
- post.setEntity(multipart);
+ put.addHeader("Connection", "keep-alive");
+ put.setEntity(new InputStreamEntity(fileInputStream, contentLength, ContentType.APPLICATION_OCTET_STREAM));
- try (CloseableHttpResponse response = httpclient.execute(post)) {
+ try (CloseableHttpResponse response = httpclient.execute(put)) {
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode == HttpStatus.SC_OK || statusCode == HttpStatus.SC_NO_CONTENT) {
LOGGER.log(Level.INFO, "Response Received. - Status OK");
@@ -381,7 +387,7 @@ private static CloseableHttpClient createConnection(ProxySelector proxySelector,
HttpClientBuilder builder;
if (ProxySettings.getProxyType() != ProxySettings.DIRECT_CONNECTION
- && StringUtils.isBlank(ProxySettings.getAuthenticationUsername())
+ && StringUtils.isBlank(ProxySettings.getAuthenticationUsername())
&& ArrayUtils.isEmpty(ProxySettings.getAuthenticationPassword())
&& WinHttpClients.isWinAuthAvailable()) {
diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/json/AuthTokenRequest.java b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/json/AuthTokenRequest.java
index 6a093fee8ededfa7cb01720fd6687e6677897c74..f137818346ef2332f1ebc8e70d41bcc67737a9a2 100644
--- a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/json/AuthTokenRequest.java
+++ b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/json/AuthTokenRequest.java
@@ -34,6 +34,9 @@ public class AuthTokenRequest {
@JsonProperty("requestFileUpload")
private boolean requestFileUpload;
+ @JsonProperty("fileUploadSize")
+ private Long fileUploadSize;
+
@JsonProperty("host_id")
private String hostId;
@@ -64,6 +67,16 @@ public AuthTokenRequest setRequestFileUpload(boolean requestFileUpload) {
return this;
}
+ public Long getFileUploadSize() {
+ return fileUploadSize;
+ }
+
+ public AuthTokenRequest setFileUploadSize(Long fileUploadSize) {
+ this.fileUploadSize = fileUploadSize;
+ return this;
+ }
+
+
public String getHostId() {
return hostId;
}
diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/json/FileUploadRequest.java b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/json/FileUploadRequest.java
new file mode 100644
index 0000000000000000000000000000000000000000..48d8dc77b4705d9fcdc2dda816a802e1bead20a7
--- /dev/null
+++ b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/json/FileUploadRequest.java
@@ -0,0 +1,69 @@
+/*
+ * Autopsy Forensic Browser
+ *
+ * Copyright 2023 Basis Technology Corp.
+ * Contact: carrier <at> sleuthkit <dot> org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.basistech.df.cybertriage.autopsy.ctapi.json;
+
+import java.io.InputStream;
+
+/**
+ * Data for a file upload request.
+ */
+public class FileUploadRequest {
+
+ private String fullUrlPath;
+ private String fileName;
+ private InputStream fileInputStream;
+ private Long contentLength;
+
+ public String getFullUrlPath() {
+ return fullUrlPath;
+ }
+
+ public FileUploadRequest setFullUrlPath(String fullUrlPath) {
+ this.fullUrlPath = fullUrlPath;
+ return this;
+ }
+
+ public String getFileName() {
+ return fileName;
+ }
+
+ public FileUploadRequest setFileName(String fileName) {
+ this.fileName = fileName;
+ return this;
+ }
+
+ public InputStream getFileInputStream() {
+ return fileInputStream;
+ }
+
+ public FileUploadRequest setFileInputStream(InputStream fileInputStream) {
+ this.fileInputStream = fileInputStream;
+ return this;
+ }
+
+ public Long getContentLength() {
+ return contentLength;
+ }
+
+ public FileUploadRequest setContentLength(Long contentLength) {
+ this.contentLength = contentLength;
+ return this;
+ }
+
+}
diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java b/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java
index 2d27671b298a84b188dec8e2af2cf14d4f6e80bd..ba8bd556b101550519d4784c3926aa8fb7d60282 100644
--- a/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java
+++ b/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java
@@ -23,6 +23,7 @@
import com.basistech.df.cybertriage.autopsy.ctapi.json.AuthTokenResponse;
import com.basistech.df.cybertriage.autopsy.ctapi.json.AuthenticatedRequestData;
import com.basistech.df.cybertriage.autopsy.ctapi.json.CTCloudBean;
+import com.basistech.df.cybertriage.autopsy.ctapi.json.FileUploadRequest;
import com.basistech.df.cybertriage.autopsy.ctapi.json.LicenseInfo;
import com.basistech.df.cybertriage.autopsy.ctapi.json.MalwareResultBean.Status;
import com.basistech.df.cybertriage.autopsy.ctapi.json.MetadataUploadRequest;
@@ -112,8 +113,10 @@ private static class SharedProcessing {
//minimum file uploads left before issuing warning
private static final long LOW_UPLOADS_REMAINING = 25;
+ // min and max upload size in bytes
private static final long MIN_UPLOAD_SIZE = 1;
- private static final long MAX_UPLOAD_SIZE = 1_000_000_000;
+ private static final long MAX_UPLOAD_SIZE = 100_000_000; // 100MB
+
private static final int NUM_FILE_UPLOAD_RETRIES = 7;
private static final long FILE_UPLOAD_RETRY_SLEEP_MILLIS = 60 * 1000;
@@ -640,7 +643,7 @@ private boolean uploadFile(IngestJobState ingestJobState, String md5, long objId
}
// get auth token / file upload url
- AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(ingestJobState.getLicenseInfo().getDecryptedLicense(), true);
+ AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(ingestJobState.getLicenseInfo().getDecryptedLicense(), af.getSize());
if (StringUtils.isBlank(authTokenResponse.getFileUploadUrl())) {
throw new CTCloudException(CTCloudException.ErrorCode.NETWORK_ERROR);
} else if (remaining(authTokenResponse.getFileUploadLimit(), authTokenResponse.getFileUploadCount()) <= 0) {
@@ -658,7 +661,13 @@ private boolean uploadFile(IngestJobState ingestJobState, String md5, long objId
// upload bytes
ReadContentInputStream fileInputStream = new ReadContentInputStream(af);
- ctApiDAO.uploadFile(authTokenResponse.getFileUploadUrl(), af.getName(), fileInputStream);
+
+ ctApiDAO.uploadFile(new FileUploadRequest()
+ .setContentLength(af.getSize())
+ .setFileInputStream(fileInputStream)
+ .setFileName(af.getName())
+ .setFullUrlPath(authTokenResponse.getFileUploadUrl())
+ );
// upload metadata
MetadataUploadRequest metaRequest = new MetadataUploadRequest()