diff --git a/bindings/java/ivy.xml b/bindings/java/ivy.xml
index af9f85d09f8d67a0bc96cab2f3647f211098b736..e70fc02a89891d2e3a9aebfc76ae2b5c4b6f5106 100644
--- a/bindings/java/ivy.xml
+++ b/bindings/java/ivy.xml
@@ -20,6 +20,9 @@
</dependency>
<dependency org="com.mchange" name="c3p0" rev="0.9.5" />
<dependency org="com.zaxxer" name="SparseBitSet" rev="1.1" />
+
+ <dependency org="org.apache.httpcomponents" name="httpmime" rev="4.5.13"/>
+ <dependency org="org.apache.httpcomponents" name="httpclient" rev="4.5.13"/>
</dependencies>
</ivy-module>
diff --git a/bindings/java/nbproject/project.xml b/bindings/java/nbproject/project.xml
index 58ab53b9b262c0a5a2414f1bbbefc6a354627323..24a86c1896e50ffb3e19dbd0ed510bc9ee6dad89 100755
--- a/bindings/java/nbproject/project.xml
+++ b/bindings/java/nbproject/project.xml
@@ -114,7 +114,7 @@
<java-data xmlns="http://www.netbeans.org/ns/freeform-project-java/4">
<compilation-unit>
<package-root>src</package-root>
- <classpath mode="compile">lib;lib/diffutils-1.2.1.jar;lib/junit-4.8.2.jar;lib/postgresql-9.4-1201.jdbc41.jar;lib/c3p0-0.9.5.jar;lib/mchange-commons-java-0.2.9.jar;lib/c3p0-0.9.5-sources.jar;lib/c3p0-0.9.5-javadoc.jar;lib/joda-time-2.4.jar;lib/commons-lang3-3.0.jar;lib/guava-19.0.jar;lib/SparseBitSet-1.1.jar;lib/gson-2.8.5.jar;lib/commons-validator-1.6.jar</classpath>
+ <classpath mode="compile">lib;lib/diffutils-1.2.1.jar;lib/junit-4.8.2.jar;lib/postgresql-9.4-1201.jdbc41.jar;lib/c3p0-0.9.5.jar;lib/mchange-commons-java-0.2.9.jar;lib/c3p0-0.9.5-sources.jar;lib/c3p0-0.9.5-javadoc.jar;lib/joda-time-2.4.jar;lib/commons-lang3-3.0.jar;lib/guava-19.0.jar;lib/SparseBitSet-1.1.jar;lib/gson-2.8.5.jar;lib/commons-validator-1.6.jar;lib/httpclient-4.5.13.jar;lib/httpcore-4.4.13.jar;lib/commons-logging-1.2.jar;lib/commons-codec-1.11.jar;lib/httpmime-4.5.13.jar</classpath>
<built-to>build</built-to>
<source-level>1.8</source-level>
</compilation-unit>
diff --git a/bindings/java/src/org/sleuthkit/datamodel/AbstractFile.java b/bindings/java/src/org/sleuthkit/datamodel/AbstractFile.java
index 82776adccab51e5c539d65e76af0a2a9954654d4..a8911a3e788a2a9cce75d7406c04650c52ffdca4 100644
--- a/bindings/java/src/org/sleuthkit/datamodel/AbstractFile.java
+++ b/bindings/java/src/org/sleuthkit/datamodel/AbstractFile.java
@@ -20,7 +20,13 @@
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.io.InputStream;
import java.io.RandomAccessFile;
+import java.nio.file.FileAlreadyExistsException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
import java.sql.SQLException;
import java.sql.Statement;
import java.text.MessageFormat;
@@ -1073,13 +1079,16 @@ private synchronized void loadLocalFile() throws TskCoreException {
throw new TskCoreException("Previously failed to load file with object ID " + getId() + " from file repository.");
}
- // Copy the file from the server
- try {
- localFile = FileRepository.downloadFromFileRepository(this);
- } catch (TskCoreException ex) {
+ try (InputStream fileRepositoryStream = FileRepository.download(this)) {
+ Path localFilePath = Paths.get(FileRepository.getTempDirectory().getAbsolutePath(), this.getSha256Hash());
+ if (!Files.exists(localFilePath)) {
+ Files.copy(fileRepositoryStream, localFilePath);
+ }
+ localFile = localFilePath.toFile();
+ } catch (FileRepositoryException | IOException ex) {
// If we've failed to download from the file repository, don't try again for this session.
errorLoadingFromFileRepo = true;
- throw ex;
+ throw new TskCoreException("Failed trying to download file from repository", ex);
}
}
}
diff --git a/bindings/java/src/org/sleuthkit/datamodel/BulkExistenceEnum.java b/bindings/java/src/org/sleuthkit/datamodel/BulkExistenceEnum.java
new file mode 100755
index 0000000000000000000000000000000000000000..e43718da46b45f2a1c5bafaccccf42243b9c1dd6
--- /dev/null
+++ b/bindings/java/src/org/sleuthkit/datamodel/BulkExistenceEnum.java
@@ -0,0 +1,30 @@
+/*
+ * SleuthKit Java Bindings
+ *
+ * Copyright 2020 Basis Technology Corp.
+ * Contact: carrier <at> sleuthkit <dot> org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.sleuthkit.datamodel;
+
+/**
+ * Represents the states of existence for a file in the file repository.
+ * Given that the bulk endpoint is tolerant of dirty SHA-256 values, INVALID
+ * is an enum status for all malformed SHA-256 values.
+ */
+public enum BulkExistenceEnum {
+ TRUE,
+ FALSE,
+ INVALID;
+}
diff --git a/bindings/java/src/org/sleuthkit/datamodel/BulkExistenceResult.java b/bindings/java/src/org/sleuthkit/datamodel/BulkExistenceResult.java
new file mode 100755
index 0000000000000000000000000000000000000000..b358513926990ef1810dd4cd051dfe69b2057b97
--- /dev/null
+++ b/bindings/java/src/org/sleuthkit/datamodel/BulkExistenceResult.java
@@ -0,0 +1,49 @@
+/*
+ * SleuthKit Java Bindings
+ *
+ * Copyright 2020 Basis Technology Corp.
+ * Contact: carrier <at> sleuthkit <dot> org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.sleuthkit.datamodel;
+
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * Container for bulk existence results.
+ */
+public class BulkExistenceResult {
+
+ private Map<String, BulkExistenceEnum> files;
+
+ /**
+ * Checks the status of a file in this container. It is assumed that the
+ * file is contained within response, otherwise a null value is returned.
+ *
+ * @param file File to test
+ * @return
+ */
+ public BulkExistenceEnum getResult(AbstractFile file) {
+ if (file.getSha256Hash() == null || file.getSha256Hash().isEmpty()) {
+ return BulkExistenceEnum.INVALID;
+ }
+
+ return files.get(file.getSha256Hash());
+ }
+
+ void setFiles(Map<String, BulkExistenceEnum> files) {
+ this.files = Collections.unmodifiableMap(files);
+ }
+}
diff --git a/bindings/java/src/org/sleuthkit/datamodel/Bundle.properties b/bindings/java/src/org/sleuthkit/datamodel/Bundle.properties
index 74eeb7f4a9e9cc75adb712f49373d558e8eeb9e3..3d3265735e9b247a3914a3a55b742f0ba218ef2b 100644
--- a/bindings/java/src/org/sleuthkit/datamodel/Bundle.properties
+++ b/bindings/java/src/org/sleuthkit/datamodel/Bundle.properties
@@ -354,6 +354,6 @@ tagsFilter.displayName.text=Must be tagged
TextFilter.displayName.text=Must include text:
TypeFilter.displayName.text=Limit event types to
FileTypesFilter.displayName.text=Limit file types to
-FileRepository.downloadError.title.text=Error downloading from file repository
-FileRepository.downloadError.msg.text=Failed to download file with object ID {0} and SHA-256 hash {1}
-FileRepository.notEnabled.msg.text=File repository is not enabled
+FileRepository.error.title.text=Error from file repository
+FileRepository.downloadError.msg.text=Encountered unexpected error from the file repository. Please see logs for more information.
+FileRepository.notEnabled.msg.text=File repository is not enabled
\ No newline at end of file
diff --git a/bindings/java/src/org/sleuthkit/datamodel/Bundle.properties-MERGED b/bindings/java/src/org/sleuthkit/datamodel/Bundle.properties-MERGED
index 74eeb7f4a9e9cc75adb712f49373d558e8eeb9e3..3d3265735e9b247a3914a3a55b742f0ba218ef2b 100644
--- a/bindings/java/src/org/sleuthkit/datamodel/Bundle.properties-MERGED
+++ b/bindings/java/src/org/sleuthkit/datamodel/Bundle.properties-MERGED
@@ -354,6 +354,6 @@ tagsFilter.displayName.text=Must be tagged
TextFilter.displayName.text=Must include text:
TypeFilter.displayName.text=Limit event types to
FileTypesFilter.displayName.text=Limit file types to
-FileRepository.downloadError.title.text=Error downloading from file repository
-FileRepository.downloadError.msg.text=Failed to download file with object ID {0} and SHA-256 hash {1}
-FileRepository.notEnabled.msg.text=File repository is not enabled
+FileRepository.error.title.text=Error from file repository
+FileRepository.downloadError.msg.text=Encountered unexpected error from the file repository. Please see logs for more information.
+FileRepository.notEnabled.msg.text=File repository is not enabled
\ No newline at end of file
diff --git a/bindings/java/src/org/sleuthkit/datamodel/FileRepository.java b/bindings/java/src/org/sleuthkit/datamodel/FileRepository.java
index 6c4b456b4acc1f195adbeda028df9c0efa0a5b7a..557145a3e52f7d241fdf66c6a5d5bcd6110e26c7 100644
--- a/bindings/java/src/org/sleuthkit/datamodel/FileRepository.java
+++ b/bindings/java/src/org/sleuthkit/datamodel/FileRepository.java
@@ -18,91 +18,147 @@
*/
package org.sleuthkit.datamodel;
+import com.google.gson.Gson;
import java.io.File;
+import java.io.BufferedReader;
+import java.io.ByteArrayOutputStream;
+import java.io.FilterInputStream;
import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.nio.file.Paths;
-import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.List;
import java.util.ResourceBundle;
+import java.util.UUID;
import java.util.logging.Level;
import java.util.logging.Logger;
+import java.util.stream.Collectors;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpHead;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.client.methods.RequestBuilder;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.entity.mime.MultipartEntityBuilder;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
/**
* Class to represent a file repository.
*/
public class FileRepository {
- private static final ResourceBundle BUNDLE = ResourceBundle.getBundle("org.sleuthkit.datamodel.Bundle");
+
private static final Logger logger = Logger.getLogger(FileRepository.class.getName());
- private static FileRepositoryErrorHandler errorHandler;
+ private static final ResourceBundle BUNDLE = ResourceBundle.getBundle("org.sleuthkit.datamodel.Bundle");
+ private static final int MAX_BULK_QUERY_SIZE = 500;
+ private final static String V1_FILES_TEMPLATE = "http://%s:%s/v1/files/";
- private final static String FILE_PATH = "v1/files/";
+ private final Gson gson;
private final FileRepositorySettings settings;
private final File fileDownloadFolder;
+ private static FileRepositoryErrorHandler errorHandler;
private static FileRepository instance;
- /**
- * Create the file repository.
- *
- * @param settings The file repository settings
- * @param fileDownloadPath The temporary folder to download files to from the repository
- */
- private FileRepository(FileRepositorySettings settings, File fileDownloadPath) {
- this.settings = settings;
- this.fileDownloadFolder = fileDownloadPath;
- }
-
- /**
- * Initializes the file repository.
- *
- * @param settings The file repository settings
- * @param fileDownloadPath The temporary folder to download files to from the repository
+ /**
+ * Create the file repository.
+ *
+ * @param settings The file repository settings
+ * @param fileDownloadPath The temporary folder to download files to from the repository
*/
- public static synchronized void initialize(FileRepositorySettings settings, File fileDownloadPath) {
- // If the download path is changing, delete any files in the old one
- if ((instance != null) && (instance.fileDownloadFolder != null)
- && ( ! instance.fileDownloadFolder.equals(fileDownloadPath))) {
- deleteDownloadFolder(instance.fileDownloadFolder);
- }
- instance = new FileRepository(settings, fileDownloadPath);
+ private FileRepository(FileRepositorySettings settings, File fileDownloadPath) {
+ this.settings = settings;
+ this.fileDownloadFolder = fileDownloadPath;
+ this.gson = new Gson();
}
- /**
- * De-initializes the file repository.
- */
- public static synchronized void deinitialize() {
- if (instance != null) {
- // Delete the temp folder
- deleteDownloadFolder(instance.fileDownloadFolder);
- }
-
- instance = null;
+ /**
+ * Delete the folder of downloaded files.
+ */
+ private static synchronized void deleteDownloadFolder(File dirPath) {
+ if (dirPath.isDirectory() == false || dirPath.exists() == false) {
+ return;
+ }
+
+ File[] files = dirPath.listFiles();
+ if (files != null) {
+ for (File file : files) {
+ if (file.isDirectory()) {
+ deleteDownloadFolder(file);
+ } else {
+ if (file.delete() == false) {
+ logger.log(Level.WARNING, "Failed to delete file {0}", file.getPath()); //NON-NLS
+ }
+ }
+ }
+ }
+ if (dirPath.delete() == false) {
+ logger.log(Level.WARNING, "Failed to delete the empty directory at {0}", dirPath.getPath()); //NON-NLS
+ }
}
-
+
+ /**
+ * Initializes the file repository.
+ *
+ * @param settings The file repository settings
+ * @param fileDownloadPath The temporary folder to download files to from the repository
+ */
+ public static synchronized void initialize(FileRepositorySettings settings, File fileDownloadPath) {
+ // If the download path is changing, delete any files in the old one
+ if ((instance != null) && (instance.fileDownloadFolder != null)
+ && ( ! instance.fileDownloadFolder.equals(fileDownloadPath))) {
+ deleteDownloadFolder(instance.fileDownloadFolder);
+ }
+ instance = new FileRepository(settings, fileDownloadPath);
+ }
+
+ /**
+ * De-initializes the file repository.
+ */
+ public static synchronized void deinitialize() {
+ if (instance != null) {
+ // Delete the temp folder
+ deleteDownloadFolder(instance.fileDownloadFolder);
+ }
+
+ instance = null;
+ }
+
/**
* Check if the file repository is enabled.
- *
+ *
* @return true if enabled, false otherwise.
*/
public static boolean isEnabled() {
return instance != null;
}
-
+
/**
* Set the error handling callback.
- *
+ *
* @param handler The error handler.
*/
public static synchronized void setErrorHandler(FileRepositoryErrorHandler handler) {
errorHandler = handler;
}
+ static synchronized File getTempDirectory() throws FileRepositoryException {
+ ensureInstanceIsEnabled();
+ return instance.fileDownloadFolder;
+ }
+
/**
- * Report an error to the user.
- * The idea is to use this for cases where it's a user error that may be able
- * to be corrected through changing the repository settings.
- *
+ * Report an error to the user. The idea is to use this for cases where it's
+ * a user error that may be able to be corrected through changing the
+ * repository settings.
+ *
* @param errorTitle The title for the error display.
* @param errorStr The error message.
*/
@@ -111,205 +167,326 @@ private static synchronized void reportError(String errorTitle, String errorStr)
errorHandler.displayErrorToUser(errorTitle, errorStr);
}
}
-
+
/**
- * Delete the folder of downloaded files.
+ * Download a file's data from the file repository. The resulting stream
+ * must be closed and it should be read as soon as possible.
+ *
+ * @param abstractFile The file to be downloaded.
+ *
+ * @return The file contents, as a stream.
+ *
+ * @throws org.sleuthkit.datamodel.FileRepositoryException
+ * @throws java.io.IOException
+ *
*/
- private static synchronized void deleteDownloadFolder(File dirPath) {
- if (dirPath.isDirectory() == false || dirPath.exists() == false) {
- return;
- }
-
- File[] files = dirPath.listFiles();
- if (files != null) {
- for (File file : files) {
- if (file.isDirectory()) {
- deleteDownloadFolder(file);
- } else {
- if (file.delete() == false) {
- logger.log(Level.WARNING, "Failed to delete file {0}", file.getPath()); //NON-NLS
- }
- }
- }
- }
- if (dirPath.delete() == false) {
- logger.log(Level.WARNING, "Failed to delete the empty directory at {0}", dirPath.getPath()); //NON-NLS
- }
+ public static synchronized InputStream download(AbstractFile abstractFile) throws FileRepositoryException, IOException {
+ // Preconditions
+ ensureInstanceIsEnabled();
+ ensureNonEmptySHA256(abstractFile);
+ ensureFileLocationIsRemote(abstractFile);
+
+ return instance.sendDownloadRequest(abstractFile.getSha256Hash());
}
-
+
/**
- * Download a file from the file repository.
- * The caller must ensure that this is not called on the same file multiple times concurrently.
- *
- * @param abstractFile The file being downloaded.
- *
- * @return The downloaded file.
- *
- * @throws TskCoreException
- */
- public static synchronized File downloadFromFileRepository(AbstractFile abstractFile) throws TskCoreException {
-
- if (instance == null) {
- String title = BUNDLE.getString("FileRepository.downloadError.title.text");
- String msg = BUNDLE.getString("FileRepository.notEnabled.msg.text");
- reportError(title, msg);
- throw new TskCoreException("File repository is not enabled");
+ * Private function to perform file download.
+ */
+ private InputStream sendDownloadRequest(String SHA256) throws IOException, FileRepositoryException {
+ final CloseableHttpClient httpClient = HttpClients.createDefault();
+ final String downloadURL = settings.createBaseURL(V1_FILES_TEMPLATE) + SHA256;
+
+ final HttpGet downloadRequest = new HttpGet(downloadURL);
+ final CloseableHttpResponse response = httpClient.execute(downloadRequest);
+ final int statusCode = response.getStatusLine().getStatusCode();
+
+ if (statusCode != HttpStatus.SC_OK) {
+ FileRepositoryException repoEx = null;
+ try {
+ final String title = BUNDLE.getString("FileRepository.error.title.text");
+ final String message = BUNDLE.getString("FileRepository.downloadError.msg.text");
+ reportError(title, message);
+ final String errorMessage = extractErrorMessage(response);
+ repoEx = new FileRepositoryException(String.format("Request "
+ + "failed with the following response body %s. Please "
+ + "check the file repository logs for more information.", errorMessage));
+ } finally {
+ try {
+ response.close();
+ } catch (IOException ex) {
+ // Best effort
+ if (repoEx != null) {
+ repoEx.addSuppressed(ex);
+ }
+ }
+
+ try {
+ httpClient.close();
+ } catch (IOException ex) {
+ // Best effort
+ if (repoEx != null) {
+ repoEx.addSuppressed(ex);
+ }
+ }
+ }
+
+ throw repoEx;
}
-
- if (! abstractFile.getFileLocation().equals(TskData.FileLocation.REPOSITORY)) {
- throw new TskCoreException("File with object ID " + abstractFile.getId() + " is not stored in the file repository");
+
+ // Client and response will close once the stream has been
+ // consumed and closed by the client.
+ return new HTTPInputStream(httpClient, response);
+ }
+
+ /**
+ * Uploads a stream of data to the file repository.
+ *
+ *
+ * @param stream Arbitrary data to store in this file repository.
+ *
+ * @throws java.io.IOException
+ * @throws org.sleuthkit.datamodel.FileRepositoryException
+ */
+ public static synchronized void upload(InputStream stream) throws IOException, FileRepositoryException {
+ // Preconditions
+ ensureInstanceIsEnabled();
+
+ instance.sendUploadRequest(stream);
+ }
+
+ /**
+ * Private function to perform file upload.
+ */
+ private void sendUploadRequest(InputStream stream) throws IOException, FileRepositoryException {
+ try (CloseableHttpClient httpClient = HttpClients.createDefault()) {
+ final String uploadURL = settings.createBaseURL(V1_FILES_TEMPLATE);
+
+ // Flush the stream to a local temp file for transport.
+ final Path temp = Paths.get(fileDownloadFolder.getAbsolutePath(), UUID.randomUUID().toString());
+ Files.copy(stream, temp);
+
+ final HttpEntity fileUpload = MultipartEntityBuilder.create()
+ .addBinaryBody("file", temp.toFile())
+ .build();
+ final HttpUriRequest postRequest = RequestBuilder.post(uploadURL)
+ .setEntity(fileUpload)
+ .build();
+
+ try (CloseableHttpResponse response = httpClient.execute(postRequest)) {
+ checkSuccess(response);
+ } catch (IOException | FileRepositoryException ex) {
+ try {
+ Files.delete(temp);
+ } catch (IOException deleteEx) {
+ ex.addSuppressed(deleteEx);
+ }
+ throw ex;
+ } finally {
+ try {
+ Files.delete(temp);
+ } catch (IOException ex) {
+ // Do nothing, best effort.
+ }
+ }
}
-
- if (abstractFile.getSha256Hash() == null || abstractFile.getSha256Hash().isEmpty()) {
- throw new TskCoreException("File with object ID " + abstractFile.getId() + " has no SHA-256 hash and can not be downloaded");
+ }
+
+ /**
+ * Checks if many abstract files are stored within this file repository.
+ * This API is tolerant of files without SHA-256 values, as opposed to its
+ * overridden counterpart, which will throw an exception if not present.
+ *
+ * @param files Files to test
+ *
+ * @return An object encapsulating the response for each file.
+ *
+ * @throws java.io.IOException
+ * @throws org.sleuthkit.datamodel.FileRepositoryException
+ */
+ public static synchronized BulkExistenceResult exists(List<AbstractFile> files) throws IOException, FileRepositoryException {
+ // Preconditions
+ ensureInstanceIsEnabled();
+ ensureBulkQuerySize(files);
+
+ return instance.sendMultiExistenceQuery(new ExistenceQuery(files));
+ }
+
+ /**
+ * Private function to perform the bulk existence query.
+ */
+ private BulkExistenceResult sendMultiExistenceQuery(ExistenceQuery query) throws IOException, FileRepositoryException {
+ try (CloseableHttpClient httpClient = HttpClients.createDefault()) {
+ final String existsURL = settings.createBaseURL(V1_FILES_TEMPLATE) + "exists";
+ final String jsonString = gson.toJson(query);
+ final StringEntity jsonEntity = new StringEntity(jsonString, StandardCharsets.UTF_8);
+
+ final HttpPut bulkExistsPost = new HttpPut(existsURL);
+ bulkExistsPost.setEntity(jsonEntity);
+ bulkExistsPost.setHeader("Accept", "application/json");
+ bulkExistsPost.setHeader("Content-type", "application/json");
+
+ try (CloseableHttpResponse response = httpClient.execute(bulkExistsPost)) {
+ checkSuccess(response);
+
+ final HttpEntity entity = response.getEntity();
+ final ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream();
+ entity.writeTo(byteOutputStream);
+ final String jsonBody = new String(byteOutputStream.toByteArray(), StandardCharsets.UTF_8);
+ return gson.fromJson(jsonBody, BulkExistenceResult.class);
+ }
}
-
- // Download the file if it's not already there.
- String targetPath = Paths.get(instance.fileDownloadFolder.getAbsolutePath(), abstractFile.getSha256Hash()).toString();
- if ( ! new File(targetPath).exists()) {
- instance.downloadFile(abstractFile, targetPath);
+ }
+
+ /**
+ * Checks if the abstract file is stored within this file repository.
+ *
+ * @param file Abstract file to query
+ *
+ * @return True/False
+ *
+ * @throws IOException
+ * @throws FileRepositoryException
+ */
+ public static synchronized boolean exists(AbstractFile file) throws IOException, FileRepositoryException {
+ // Preconditions
+ ensureInstanceIsEnabled();
+ ensureNonEmptySHA256(file);
+
+ if (!file.getFileLocation().equals(TskData.FileLocation.REPOSITORY)) {
+ return false;
}
-
- // Check that we got the file.
- File tempFile = new File(targetPath);
- if (tempFile.exists()) {
- return tempFile;
- } else {
- String title = BUNDLE.getString("FileRepository.downloadError.title.text");
- String msg = MessageFormat.format(BUNDLE.getString("FileRepository.downloadError.msg.text"), abstractFile.getId(), abstractFile.getSha256Hash());
- reportError(title, msg);
- throw new TskCoreException("Failed to download file with object ID " + abstractFile.getId()
- + " and SHA-256 hash " + abstractFile.getSha256Hash() + " from file repository");
+
+ return instance.sendSingularExistenceQuery(file.getSha256Hash());
+ }
+
+ /**
+ * Private function to perform the existence query.
+ */
+ private boolean sendSingularExistenceQuery(String SHA256) throws IOException, FileRepositoryException {
+ try (CloseableHttpClient httpClient = HttpClients.createDefault()) {
+ final String existsURL = settings.createBaseURL(V1_FILES_TEMPLATE) + SHA256;
+
+ final HttpHead request = new HttpHead(existsURL);
+
+ try (CloseableHttpResponse response = httpClient.execute(request)) {
+ final int statusCode = response.getStatusLine().getStatusCode();
+ switch (statusCode) {
+ case HttpStatus.SC_OK:
+ return true;
+ case HttpStatus.SC_NOT_FOUND:
+ return false;
+ default:
+ throw new FileRepositoryException(String.format("Unexpected "
+ + "response code. Expected 200 or 404, but instead got %d. "
+ + "Please check the file repository logs for more information.", statusCode));
+ }
+ }
}
}
-
+
/**
- * Download the file.
- *
- * @param abstractFile The file being downloaded.
- * @param targetPath The location to save the file to.
- *
- * @throws TskCoreException
- */
- private void downloadFile(AbstractFile abstractFile, String targetPath) throws TskCoreException {
-
- String url = "http://" + settings.getAddress() + ":" + settings.getPort() + "/" + FILE_PATH + abstractFile.getSha256Hash();
-
- List<String> command = new ArrayList<>();
- command.add("curl");
- command.add("-X");
- command.add("GET");
- command.add(url);
- command.add("-H");
- command.add("accept: */*");
- command.add("--output");
- command.add(targetPath);
-
- ProcessBuilder processBuilder = new ProcessBuilder(command).inheritIO();
- try {
- Process process = processBuilder.start();
- process.waitFor();
- } catch (IOException ex) {
- String title = BUNDLE.getString("FileRepository.downloadError.title.text");
- String msg = MessageFormat.format(BUNDLE.getString("FileRepository.downloadError.msg.text"), abstractFile.getId(), abstractFile.getSha256Hash());
- reportError(title, msg);
- throw new TskCoreException("Error downloading file with SHA-256 hash " + abstractFile.getSha256Hash() + " from file repository", ex);
- } catch (InterruptedException ex) {
- throw new TskCoreException("Interrupted while downloading file with SHA-256 hash " + abstractFile.getSha256Hash() + " from file repository", ex);
+ * Prevents a query if the file is not remote.
+ */
+ private static void ensureFileLocationIsRemote(AbstractFile abstractFile) throws FileRepositoryException {
+ if (!abstractFile.getFileLocation().equals(TskData.FileLocation.REPOSITORY)) {
+ throw new FileRepositoryException("File with object ID " + abstractFile.getId() + " is not stored in the file repository");
}
}
-
+
/**
- * Upload a given file to the file repository.
- *
- * @param filePath The path on disk to the file being uploaded.
- *
- * @throws TskCoreException
- */
- public static synchronized void uploadToFileRepository(String filePath) throws TskCoreException {
-
- if (instance == null) {
- throw new TskCoreException("File repository is not enabled");
+ * Prevents a query for a file with no SHA-256.
+ */
+ private static void ensureNonEmptySHA256(AbstractFile abstractFile) throws FileRepositoryException {
+ if (abstractFile.getSha256Hash() == null || abstractFile.getSha256Hash().isEmpty()) {
+ throw new FileRepositoryException("File with object ID " + abstractFile.getId() + " has no SHA-256 hash.");
}
-
- File file = new File(filePath);
- if (! file.exists()) {
- throw new TskCoreException("Error uploading file " + filePath + " to file repository - file does not exist");
+ }
+
+ /**
+ * Ensures the instance is enabled, notifying users otherwise.
+ */
+ private static void ensureInstanceIsEnabled() throws FileRepositoryException {
+ if (!isEnabled()) {
+ final String title = BUNDLE.getString("FileRepository.error.title.text");
+ final String msg = BUNDLE.getString("FileRepository.notEnabled.msg.text");
+ reportError(title, msg);
+ throw new FileRepositoryException("File repository is not enabled");
}
-
- // Upload the file.
- instance.uploadFile(file);
}
-
+
/**
- * Upload the file.
- *
- * @param file The file being uploaded.
- *
- * @throws TskCoreException
- */
- private void uploadFile(File file) throws TskCoreException {
- String url = "http://" + settings.getAddress() + ":" + settings.getPort() + "/" + FILE_PATH;
-
- // Example: curl -X POST "http://localhost:8080/api/files" -H "accept: application/json" -H "Content-Type: multipart/form-data" -F "file=@Report.xml"
- List<String> command = new ArrayList<>();
- command.add("curl");
- command.add("-X");
- command.add("POST");
- command.add(url);
- command.add("-H");
- command.add("accept: application/json");
- command.add("-H");
- command.add("Content-Type: multipart/form-data");
- command.add("-F");
- command.add("file=@" + file.getAbsolutePath());
-
- ProcessBuilder processBuilder = new ProcessBuilder(command).inheritIO();
- try {
- Process process = processBuilder.start();
- process.waitFor();
- } catch (IOException | InterruptedException ex) {
- throw new TskCoreException("Error saving file at " + file.getAbsolutePath() + " to file repository", ex);
- }
- }
-
+ * Prevents a request from being made if it exceeds the maximum threshold
+ * for a bulk query.
+ */
+ private static void ensureBulkQuerySize(List<AbstractFile> files) throws FileRepositoryException {
+ if (files.size() > MAX_BULK_QUERY_SIZE) {
+ throw new FileRepositoryException(String.format("Exceeds the allowable "
+ + "threshold (%d) for a single request.", MAX_BULK_QUERY_SIZE));
+ }
+ }
+
/**
- * Utility class to hold the file repository server settings.
+ * Checks the status code of the response and throws a templated exception
+ * if it's not the expected 200 code.
*/
- static public class FileRepositorySettings {
- private final String address;
- private final String port;
-
- /**
- * Create a FileRepositorySettings instance for the server.
- *
- * @param address The IP address/hostname of the server.
- * @param port The port.
- */
- public FileRepositorySettings(String address, String port) {
- this.address = address;
- this.port = port;
+ private static void checkSuccess(CloseableHttpResponse response) throws FileRepositoryException, IOException {
+ final int statusCode = response.getStatusLine().getStatusCode();
+
+ if (statusCode != HttpStatus.SC_OK) {
+ final String errorMessage = extractErrorMessage(response);
+ throw new FileRepositoryException(String.format("Request failed with "
+ + "the following response body %s. Please check the file "
+ + "repository logs for more information.", errorMessage));
}
-
- String getAddress() {
- return address;
+ }
+
+ /**
+ * Extracts the entire response body as a plain string.
+ */
+ private static String extractErrorMessage(CloseableHttpResponse response) throws IOException {
+ try (BufferedReader reader = new BufferedReader(new InputStreamReader(
+ response.getEntity().getContent(), StandardCharsets.UTF_8))) {
+ return reader.lines().collect(Collectors.joining("\n"));
}
-
- String getPort() {
- return port;
+ }
+
+ /**
+ * Query object to be serialized by GSON and sent as a payload to the bulk
+ * exists endpoint.
+ */
+ private static class ExistenceQuery {
+
+ private final List<String> files;
+
+ ExistenceQuery(List<AbstractFile> absFiles) {
+ files = new ArrayList<>();
+ for (AbstractFile file : absFiles) {
+ if (file.getSha256Hash() != null && !file.getSha256Hash().isEmpty()) {
+ files.add(file.getSha256Hash());
+ }
+ }
}
}
-
+
/**
- * Callback class to use for error reporting.
+ * Streams data over a HTTP connection.
*/
- public interface FileRepositoryErrorHandler {
- /**
- * Handles displaying an error message to the user (if appropriate).
- *
- * @param title The title for the error display.
- * @param error The more detailed error message to display.
- */
- void displayErrorToUser(String title, String error);
+ private static class HTTPInputStream extends FilterInputStream {
+
+ private final CloseableHttpResponse response;
+ private final CloseableHttpClient client;
+
+ HTTPInputStream(CloseableHttpClient client, CloseableHttpResponse response) throws IOException {
+ super(response.getEntity().getContent());
+ this.response = response;
+ this.client = client;
+ }
+
+ @Override
+ public void close() throws IOException {
+ super.close();
+ response.close();
+ client.close();
+ }
}
}
diff --git a/bindings/java/src/org/sleuthkit/datamodel/FileRepositoryErrorHandler.java b/bindings/java/src/org/sleuthkit/datamodel/FileRepositoryErrorHandler.java
new file mode 100755
index 0000000000000000000000000000000000000000..341a6f4b830b17a68043b56f8137cc3d84cd5e61
--- /dev/null
+++ b/bindings/java/src/org/sleuthkit/datamodel/FileRepositoryErrorHandler.java
@@ -0,0 +1,34 @@
+/*
+ * SleuthKit Java Bindings
+ *
+ * Copyright 2020 Basis Technology Corp.
+ * Contact: carrier <at> sleuthkit <dot> org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.sleuthkit.datamodel;
+
+/**
+ * Callback class to use for error reporting.
+ */
+public interface FileRepositoryErrorHandler {
+
+ /**
+ * Handles displaying an error message to the user (if appropriate).
+ *
+ * @param title The title for the error display.
+ * @param error The more detailed error message to display.
+ */
+ void displayErrorToUser(String title, String error);
+
+}
diff --git a/bindings/java/src/org/sleuthkit/datamodel/FileRepositoryException.java b/bindings/java/src/org/sleuthkit/datamodel/FileRepositoryException.java
new file mode 100755
index 0000000000000000000000000000000000000000..df66e7a7ac3c999ac0aee58902666aa6a6367683
--- /dev/null
+++ b/bindings/java/src/org/sleuthkit/datamodel/FileRepositoryException.java
@@ -0,0 +1,28 @@
+/*
+ * SleuthKit Java Bindings
+ *
+ * Copyright 2020 Basis Technology Corp.
+ * Contact: carrier <at> sleuthkit <dot> org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.sleuthkit.datamodel;
+
+public class FileRepositoryException extends Exception {
+
+ private static final long serialVersionUID = 1L;
+
+ public FileRepositoryException(String msg) {
+ super(msg);
+ }
+}
diff --git a/bindings/java/src/org/sleuthkit/datamodel/FileRepositorySettings.java b/bindings/java/src/org/sleuthkit/datamodel/FileRepositorySettings.java
new file mode 100755
index 0000000000000000000000000000000000000000..8b64881f786c30a9ef4e4bd7f3dbb61c0252c1f0
--- /dev/null
+++ b/bindings/java/src/org/sleuthkit/datamodel/FileRepositorySettings.java
@@ -0,0 +1,46 @@
+/*
+ * SleuthKit Java Bindings
+ *
+ * Copyright 2020 Basis Technology Corp.
+ * Contact: carrier <at> sleuthkit <dot> org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.sleuthkit.datamodel;
+
+/**
+ * Utility class to hold the file repository server settings.
+ */
+public class FileRepositorySettings {
+
+ private final String address;
+ private final String port;
+
+ /**
+ * Create a FileRepositorySettings instance for the server.
+ *
+ * @param address The IP address/hostname of the server.
+ * @param port The port.
+ */
+ public FileRepositorySettings(String address, String port) {
+ this.address = address;
+ this.port = port;
+ }
+
+ /**
+ * Fills in an API template with the address and port.
+ */
+ String createBaseURL(String urlTemplate) {
+ return String.format(urlTemplate, address, port);
+ }
+}