diff --git a/bindings/java/src/org/sleuthkit/datamodel/AbstractFile.java b/bindings/java/src/org/sleuthkit/datamodel/AbstractFile.java
index 09e397c7a2f802b7bd780591de8e06de8e261674..e64010438c3006911324fa51961301dca0827577 100644
--- a/bindings/java/src/org/sleuthkit/datamodel/AbstractFile.java
+++ b/bindings/java/src/org/sleuthkit/datamodel/AbstractFile.java
@@ -1055,7 +1055,7 @@ private synchronized void loadLocalFile() throws TskCoreException {
}
} else {
// Copy the file from the server
- localFile = getSleuthkitCase().loadFromFileService(this);
+ localFile = getSleuthkitCase().getFileRepositoryManager().loadFromFileRepository(this);
}
}
diff --git a/bindings/java/src/org/sleuthkit/datamodel/FileRepositoryManager.java b/bindings/java/src/org/sleuthkit/datamodel/FileRepositoryManager.java
new file mode 100644
index 0000000000000000000000000000000000000000..eb523c5570d8b70a32bfd57f91ff7b0f7a37f867
--- /dev/null
+++ b/bindings/java/src/org/sleuthkit/datamodel/FileRepositoryManager.java
@@ -0,0 +1,257 @@
+/*
+ * SleuthKit Java Bindings
+ *
+ * Copyright 2020 Basis Technology Corp.
+ * Contact: carrier <at> sleuthkit <dot> org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.sleuthkit.datamodel;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Paths;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import org.sleuthkit.datamodel.SleuthkitCase.CaseDbConnection;
+
+/**
+ *
+ */
+public class FileRepositoryManager {
+
+ private final static String FILE_PATH = "v1/files/";
+ private final SleuthkitCase skCase;
+ private FileRepositorySettings settings;
+ private String fileDownloadFolder;
+
+ // Create an unitialized file repository
+ FileRepositoryManager(SleuthkitCase skCase) {
+ this.skCase = skCase;
+ settings = null;
+ fileDownloadFolder = "";
+ }
+
+ public void initializeSettings(FileRepositorySettings settings, String fileDownloadPath) {
+ this.settings = settings;
+ this.fileDownloadFolder = fileDownloadPath;
+ }
+
+ /**
+ * Check whether the file repository has been initialized.
+ */
+ public boolean isEnabled() {
+ return settings != null;
+ }
+
+
+ public boolean caseUsesFileRepository() throws TskCoreException {
+ skCase.acquireSingleUserCaseReadLock();
+ try (CaseDbConnection connection = skCase.getConnection();
+ Statement statement = connection.createStatement();
+ ResultSet rs = connection.executeQuery(statement, "SELECT COUNT(*) as count FROM tsk_files WHERE location=" + TskData.FileLocation.REPOSITORY.getValue());) {
+ int count = 0;
+ if (rs.next()) {
+ count = rs.getInt("count");
+ }
+ return count > 0;
+ } catch (SQLException ex) {
+ throw new TskCoreException("Error querying case database for files stored in repository", ex);
+ } finally {
+ skCase.releaseSingleUserCaseReadLock();
+ }
+ }
+
+ private String getFilePath(AbstractFile abstractFile) {
+ return Paths.get(fileDownloadFolder, abstractFile.getSha256Hash()).toString();
+ }
+
+ public File loadFromFileRepository(AbstractFile abstractFile) throws TskCoreException {
+ if (!isEnabled()) {
+ throw new TskCoreException("File repository is not enabled");
+ }
+
+ if (! abstractFile.getFileLocation().equals(TskData.FileLocation.REPOSITORY)) {
+ throw new TskCoreException("File with object ID " + abstractFile.getId() + " is not stored in the file repository");
+ }
+
+ if (abstractFile.getSha256Hash() == null || abstractFile.getSha256Hash().isEmpty()) {
+ throw new TskCoreException("File with object ID " + abstractFile.getId() + " has no SHA-256 hash and can not be downloaded");
+ }
+
+ // Download the file if it's not already there.
+ String downloadPath = getFilePath(abstractFile);
+ if ( ! new File(downloadPath).exists()) {
+ downloadFileFromFileService(abstractFile, downloadPath);
+ }
+
+ // Check that we got the file.
+ File tempFile = new File(downloadPath);
+ if (tempFile.exists()) {
+ return tempFile;
+ } else {
+ throw new TskCoreException("Failed to download file with object ID " + abstractFile.getId()
+ + " and SHA-256 hash " + abstractFile.getSha256Hash() + " from file repository");
+ }
+ }
+
+ private String makeUrl(AbstractFile abstractFile) {
+ return "http://" + settings.getAddress() + ":" + settings.getPort() + "/" + FILE_PATH + abstractFile.getSha256Hash();
+ }
+
+ private void downloadFileFromFileService(AbstractFile abstractFile, String downloadPath) throws TskCoreException {
+
+ List<String> command = new ArrayList<>();
+ command.add("curl");
+ command.add("-X");
+ command.add("GET");
+ command.add(makeUrl(abstractFile));
+ command.add("-H");
+ command.add("accept: */*");
+ command.add("--output");
+ command.add(downloadPath);
+
+ ProcessBuilder processBuilder = new ProcessBuilder(command).inheritIO();
+ try {
+ Process process = processBuilder.start();
+ process.waitFor();
+ } catch (IOException | InterruptedException ex) {
+ throw new TskCoreException("Error downloading file with SHA-256 hash " + abstractFile.getSha256Hash() + " from file repository", ex);
+ }
+ }
+
+ /**
+ * @param abstractFile
+ * @param trans
+ */
+ public void saveToFileRepository(AbstractFile abstractFile, SleuthkitCase.CaseDbTransaction trans) throws TskCoreException {
+
+ if (! isEnabled()) {
+ throw new TskCoreException("File repository is not enabled");
+ }
+
+ // Make sure the SHA-256 hash has been calculated
+ if (abstractFile.getSha256Hash() == null || abstractFile.getSha256Hash().isEmpty()) {
+ HashUtility.calculateHashes(abstractFile, Arrays.asList(HashUtility.HashType.SHA256));
+ abstractFile.save();
+ }
+
+ String filePath = "";
+ if (abstractFile.getLocalPath() == null || abstractFile.getLocalPath().isEmpty()) {
+ try {
+ filePath = extractFileToDisk(abstractFile);
+ } catch (IOException ex) {
+ throw new TskCoreException("Error writing temporary file to disk", ex);
+ }
+ } else {
+ filePath = abstractFile.getLocalAbsPath();
+ }
+
+ // Save the abstractFile data
+ saveLocalFileToFileService(filePath);
+
+ // Update the file table entry
+ try {
+ SleuthkitCase.CaseDbConnection connection = trans.getConnection();
+
+ // Change the location to REPOSITORY
+ String updateLocation = "UPDATE tsk_files SET location = ? WHERE obj_id = ?"; // NON-NLS
+ PreparedStatement statement = connection.getPreparedStatement(updateLocation, Statement.NO_GENERATED_KEYS);
+ statement.clearParameters();
+ statement.setLong(1, TskData.FileLocation.REPOSITORY.getValue());
+ statement.setLong(2, abstractFile.getId());
+ connection.executeUpdate(statement);
+
+ // Remove entry for this file in tsk_files_path (if it exists)
+ String removePath = "DELETE FROM tsk_files_path WHERE obj_id = ?"; // NON-NLS
+ statement = connection.getPreparedStatement(removePath, Statement.NO_GENERATED_KEYS);
+ statement.clearParameters();
+ statement.setLong(1, abstractFile.getId());
+ connection.executeUpdate(statement);
+
+ } catch (SQLException ex) {
+ throw new TskCoreException("Error updating database for move to file repository", ex);
+ }
+ }
+
+ private void saveLocalFileToFileService(String pathToFile) throws TskCoreException {
+ File file = new File(pathToFile);
+
+ // curl -X POST "http://localhost:8080/api/files" -H "accept: application/json" -H "Content-Type: multipart/form-data" -F "file=@Report.xml"
+ List<String> command = new ArrayList<>();
+ command.add("curl");
+ command.add("-X");
+ command.add("POST");
+ command.add("http://localhost:8080/v1/files");
+ command.add("-H");
+ command.add("accept: application/json");
+ command.add("-H");
+ command.add("Content-Type: multipart/form-data");
+ command.add("-F");
+ command.add("file=@" + file.getAbsolutePath());
+
+ ProcessBuilder processBuilder = new ProcessBuilder(command).inheritIO();
+ try {
+ Process process = processBuilder.start();
+ process.waitFor();
+ } catch (IOException | InterruptedException ex) {
+ throw new TskCoreException("Error saving file at " + pathToFile + " to file repository", ex);
+ }
+ }
+
+ private String extractFileToDisk(AbstractFile abstractFile) throws IOException {
+ String extractedPath = getFilePath(abstractFile);
+
+ InputStream in = new ReadContentInputStream(abstractFile);
+ try (FileOutputStream out = new FileOutputStream(extractedPath, false)) {
+ byte[] buffer = new byte[0x2000];
+ int len = in.read(buffer);
+ while (len != -1) {
+ out.write(buffer, 0, len);
+ len = in.read(buffer);
+ }
+ } catch (FileNotFoundException ex) {
+ throw new IOException("Error exporting file with object ID " + abstractFile.getId() + " to " + extractedPath, ex);
+ } finally {
+ in.close();
+ }
+ return extractedPath;
+ }
+
+
+ static public class FileRepositorySettings {
+ String address;
+ String port;
+
+ public FileRepositorySettings(String address, String port) {
+ this.address = address;
+ this.port = port;
+ }
+
+ public String getAddress() {
+ return address;
+ }
+
+ public String getPort() {
+ return port;
+ }
+ }
+}
diff --git a/bindings/java/src/org/sleuthkit/datamodel/SleuthkitCase.java b/bindings/java/src/org/sleuthkit/datamodel/SleuthkitCase.java
index d8923b96da59ae55393556f65b1559b153b9c220..d601a67c86d4266a73838bd43c48a1651e0583c4 100644
--- a/bindings/java/src/org/sleuthkit/datamodel/SleuthkitCase.java
+++ b/bindings/java/src/org/sleuthkit/datamodel/SleuthkitCase.java
@@ -210,6 +210,7 @@ public class SleuthkitCase {
private Blackboard blackboard;
private CaseDbAccessManager dbAccessManager;
private TaggingManager taggingMgr;
+ private FileRepositoryManager fileRepositoryManager;
private final Map<String, Set<Long>> deviceIdToDatasourceObjIdMap = new HashMap<>();
@@ -388,6 +389,7 @@ private void init() throws Exception {
timelineMgr = new TimelineManager(this);
dbAccessManager = new CaseDbAccessManager(this);
taggingMgr = new TaggingManager(this);
+ fileRepositoryManager = new FileRepositoryManager(this);
}
/**
@@ -499,6 +501,15 @@ public synchronized CaseDbAccessManager getCaseDbAccessManager() throws TskCoreE
public synchronized TaggingManager getTaggingManager() {
return taggingMgr;
}
+
+ /**
+ * Get the case database FileRepositoryManager object.
+ *
+ * @return The per case FileRepositoryManager object.
+ */
+ public synchronized FileRepositoryManager getFileRepositoryManager() {
+ return fileRepositoryManager;
+ }
/**
* Make sure the predefined artifact types are in the artifact types table.
@@ -7018,49 +7029,7 @@ public LocalFile addLocalFile(String fileName, String localPath,
String md5, String sha256, FileKnown known, String mimeType,
boolean isFile, TskData.EncodingType encodingType,
Content parent, CaseDbTransaction transaction) throws TskCoreException {
-
- return addLocalFile(fileName, localPath,
- size, ctime, crtime, atime, mtime,
- md5, sha256, known, mimeType,
- isFile, encodingType,
- parent, TskData.FileLocation.LOCAL, transaction);
- }
-
- /**
- * Adds a local/logical file to the case database. The database operations
- * are done within a caller-managed transaction; the caller is responsible
- * for committing or rolling back the transaction.
- *
- * @param fileName The name of the file.
- * @param localPath The absolute path (including the file name) of the
- * local/logical in secondary storage.
- * @param size The size of the file in bytes.
- * @param ctime The changed time of the file.
- * @param crtime The creation time of the file.
- * @param atime The accessed time of the file
- * @param mtime The modified time of the file.
- * @param md5 The MD5 hash of the file.
- * @param sha256 The SHA-256 hash of the file.
- * @param known The known status of the file (can be null)
- * @param mimeType The MIME type of the file
- * @param isFile True, unless the file is a directory.
- * @param encodingType Type of encoding used on the file
- * @param parent The parent of the file (e.g., a virtual directory)
- * @param location The location of the file.
- * @param transaction A caller-managed transaction within which the add
- * file operations are performed.
- *
- * @return An object representing the local/logical file.
- *
- * @throws TskCoreException if there is an error completing a case database
- * operation.
- */
- LocalFile addLocalFile(String fileName, String localPath,
- long size, long ctime, long crtime, long atime, long mtime,
- String md5, String sha256, FileKnown known, String mimeType,
- boolean isFile, TskData.EncodingType encodingType,
- Content parent, TskData.FileLocation location, CaseDbTransaction transaction) throws TskCoreException {
-
+
CaseDbConnection connection = transaction.getConnection();
Statement queryStatement = null;
try {
@@ -7120,7 +7089,7 @@ LocalFile addLocalFile(String fileName, String localPath,
dataSourceObjId = getDataSourceObjectId(connection, parent.getId());
}
statement.setString(19, parentPath);
- statement.setLong(20, location.getValue());
+ statement.setLong(20, TskData.FileLocation.LOCAL.getValue());
statement.setLong(21, dataSourceObjId);
final String extension = extractExtension(fileName);
statement.setString(22, extension);
@@ -7431,132 +7400,6 @@ private void updateFilePath(CaseDbConnection connection, long objId, String path
statement.setLong(3, objId);
connection.executeUpdate(statement);
}
-
- // TEMP TEMP
- // Should be set at the same time as the file service server
- File getFileServiceTempFolder() {
- return new File("R:\\mytemp");
- }
-
- File loadFromFileService(AbstractFile abstractFile) throws TskCoreException {
- if (! abstractFile.getFileLocation().equals(TskData.FileLocation.MICROSERVICE)) {
- throw new TskCoreException("Not file service");
- }
-
- // TODO verify there's a hash (there should be)
-
- // TODO check existence first?
- String downloadPath = Paths.get(getFileServiceTempFolder().toString(), abstractFile.getSha256Hash()).toString();
- downloadFileFromFileService(abstractFile, downloadPath);
-
- File tempFile = new File(downloadPath);
- if (tempFile.exists()) {
- // YAY!
- return tempFile;
- } else {
- System.out.println("\n### oh no... " + tempFile.toString() + " does not exist");
- throw new TskCoreException("error");
- }
-
- }
-
- private void downloadFileFromFileService(AbstractFile file, String downloadPath) {
-
- // curl -X GET "http://localhost:8080/api/files/(hash value)" -H "accept: */*" --output downloadPath
- List<String> command = new ArrayList<>();
- command.add("curl");
- command.add("-X");
- command.add("GET");
- command.add("http://localhost:8080/v1/files/" + file.getSha256Hash());
- command.add("-H");
- command.add("accept: */*");
- command.add("--output");
- command.add(downloadPath);
-
- System.out.println("### Process builder commands");
- for(String s:command){
- System.out.println(" " + s);
- }
-
- ProcessBuilder processBuilder = new ProcessBuilder(command).inheritIO();
- try {
- Process process = processBuilder.start();
- process.waitFor();
- } catch (IOException | InterruptedException ex) {
- ex.printStackTrace();
- }
-
-
- }
-
- /**
- * TODO: should be abstractFile or content?
- * @param abstractFile
- */
- public void saveToFileService(AbstractFile abstractFile, CaseDbTransaction trans) throws TskCoreException {
-
- if (abstractFile.getLocalPath() == null || abstractFile.getLocalPath().isEmpty()) {
- // TODO
- System.out.println("Not supported yet\n");
- } else {
- // Make sure the SHA-256 hash has been calculated
- if (abstractFile.getSha256Hash() == null || abstractFile.getSha256Hash().isEmpty()) {
- HashUtility.calculateHashes(abstractFile, Arrays.asList(HashUtility.HashType.SHA256));
- abstractFile.save();
- }
-
- // Save the abstractFile data
- saveLocalFileToFileService(abstractFile);
-
- // Update the file table entry
- try {
- CaseDbConnection connection = trans.getConnection();
- PreparedStatement statement = connection.getPreparedStatement(PREPARED_STATEMENT.UPDATE_FILE_SERVICE_FILE);
- statement.clearParameters();
-
- statement.setLong(1, TskData.FileLocation.MICROSERVICE.getValue());
- statement.setLong(2, abstractFile.getId());
-
- connection.executeUpdate(statement);
-
- // TODO remove local path
-
- } catch (SQLException ex) {
- ex.printStackTrace();
- }
- }
-
- }
-
- private void saveLocalFileToFileService(AbstractFile abstractFile) {
- File file = new File(abstractFile.getLocalPath());
-
- // curl -X POST "http://localhost:8080/api/files" -H "accept: application/json" -H "Content-Type: multipart/form-data" -F "file=@Report.xml"
- List<String> command = new ArrayList<>();
- command.add("curl");
- command.add("-X");
- command.add("POST");
- command.add("http://localhost:8080/v1/files");
- command.add("-H");
- command.add("accept: application/json");
- command.add("-H");
- command.add("Content-Type: multipart/form-data");
- command.add("-F");
- command.add("file=@" + file.getAbsolutePath());
-
- System.out.println("### Process builder commands");
- for(String s:command){
- System.out.println(" " + s);
- }
-
- ProcessBuilder processBuilder = new ProcessBuilder(command).inheritIO();
- try {
- Process process = processBuilder.start();
- process.waitFor();
- } catch (IOException | InterruptedException ex) {
- ex.printStackTrace();
- }
- }
/**
* Find all files in the data source, by name and parent
@@ -11517,8 +11360,7 @@ private enum PREPARED_STATEMENT {
INSERT_FS_INFO("INSERT INTO tsk_fs_info (obj_id, data_source_obj_id, img_offset, fs_type, block_size, block_count, root_inum, first_inum, last_inum, display_name)"
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"),
SELECT_TAG_NAME_BY_ID("SELECT * FROM tag_names where tag_name_id = ?"),
- SELECT_TAG_NAME_BY_NAME("SELECT * FROM tag_names where display_name = ?"),
- UPDATE_FILE_SERVICE_FILE("UPDATE tsk_files SET location = ? WHERE obj_id = ?");
+ SELECT_TAG_NAME_BY_NAME("SELECT * FROM tag_names where display_name = ?");
private final String sql;
diff --git a/bindings/java/src/org/sleuthkit/datamodel/TskData.java b/bindings/java/src/org/sleuthkit/datamodel/TskData.java
index e83310ba21e83e924bf31a0b6842ac65ad2fff72..b49e36a201a65d412e79358c85cf5d95d32153fe 100644
--- a/bindings/java/src/org/sleuthkit/datamodel/TskData.java
+++ b/bindings/java/src/org/sleuthkit/datamodel/TskData.java
@@ -884,7 +884,7 @@ public static EncodingType valueOf(int type) {
*/
public enum FileLocation{
LOCAL(0),
- MICROSERVICE(1);
+ REPOSITORY(1);
private final int value;