diff --git a/.gitignore b/.gitignore
old mode 100755
new mode 100644
diff --git a/tools/fiwalk/src/content.cpp b/tools/fiwalk/src/content.cpp
index fb8c6f2fde137515d040e74b6dac7241c5cbbbdd..9baf37ec061ab9246010a81f9da72696bf636066 100644
--- a/tools/fiwalk/src/content.cpp
+++ b/tools/fiwalk/src/content.cpp
@@ -53,6 +53,9 @@
#include <err.h>
#endif
+#define MAX_SPARSE_SIZE 1024*1024*64
+
+
/****************************************************************
** Content Output
****************************************************************/
@@ -252,32 +255,36 @@ void content::write_record()
for(seglist::const_iterator i = this->segs.begin();i!=this->segs.end();i++){
char buf[1024];
if(i->flags & TSK_FS_BLOCK_FLAG_SPARSE){
- sprintf(buf," <byte_run file_offset='%"PRIu64"' fill='0' len='%"PRIu64"'/>\n",
- i->file_offset,i->len);
+ sprintf(buf," <byte_run file_offset='%"PRIu64"' fill='0' len='%"PRIu64"'", i->file_offset,i->len);
} else if (i->flags & TSK_FS_BLOCK_FLAG_RAW){
sprintf(buf,
- " <byte_run file_offset='%"PRIu64"' fs_offset='%"PRIu64"' " "img_offset='%"PRIu64"' len='%"PRIu64"'/>\n",
+ " <byte_run file_offset='%"PRIu64"' fs_offset='%"PRIu64"' " "img_offset='%"PRIu64"' len='%"PRIu64"'",
i->file_offset,i->fs_offset,i->img_offset,i->len);
} else if (i->flags & TSK_FS_BLOCK_FLAG_COMP){
if(i->fs_offset){
sprintf(buf,
" <byte_run file_offset='%"PRIu64"' fs_offset='%"PRIu64"' "
- "img_offset='%"PRIu64"' uncompressed_len='%"PRIu64"'/>\n",
+ "img_offset='%"PRIu64"' uncompressed_len='%"PRIu64"'",
i->file_offset,i->fs_offset,i->img_offset,i->len);
} else {
sprintf(buf,
- " <byte_run file_offset='%"PRIu64"' uncompressed_len='%"PRIu64"'/>\n",
- i->file_offset,i->len);
+ " <byte_run file_offset='%"PRIu64"' uncompressed_len='%"PRIu64"'", i->file_offset,i->len);
}
} else if (i->flags & TSK_FS_BLOCK_FLAG_RES){
sprintf(buf,
" <byte_run file_offset='%"PRIu64"' fs_offset='%"PRIu64"' "
- "img_offset='%"PRIu64"' len='%"PRIu64"' type='resident'/>\n",
+ "img_offset='%"PRIu64"' len='%"PRIu64"' type='resident'",
i->file_offset,i->fs_offset,i->img_offset,i->len);
} else{
- sprintf(buf," <byte_run file_offset='%"PRIu64"' unknown_flags='%d'/>\n",i->file_offset,i->flags);
+ sprintf(buf," <byte_run file_offset='%"PRIu64"' unknown_flags='%d'",i->file_offset,i->flags);
}
runs += buf;
+
+ if(i->md5.size()){
+ runs += "><hashdigest type='MD5'>" + i->md5 + "</hashdigest></byte_run>\n";
+ } else {
+ runs += "/>\n";
+ }
}
file_info_xml("byte_runs",runs);
if(!invalid){
@@ -306,14 +313,15 @@ void content::write_record()
bool content::need_file_walk()
{
return opt_md5 || opt_sha1 || opt_save || do_plugin || opt_magic
- || opt_get_fragments || opt_body_file;
-// || opt_compute_sector_hashes;
+ || opt_get_fragments || opt_body_file
+ || opt_sector_hash;
}
/** Called to create a new segment. */
void content::add_seg(int64_t img_offset,int64_t fs_offset,
int64_t file_offset,int64_t len,
- TSK_FS_BLOCK_FLAG_ENUM flags)
+ TSK_FS_BLOCK_FLAG_ENUM flags,
+ const std::string &md5)
{
seg newseg;
newseg.img_offset = img_offset;
@@ -321,6 +329,7 @@ void content::add_seg(int64_t img_offset,int64_t fs_offset,
newseg.file_offset = file_offset;
newseg.len = len;
newseg.flags = flags;
+ newseg.md5 = md5;
this->segs.push_back(newseg);
}
@@ -387,3 +396,108 @@ content::~content()
::unlink(tempfile_path.c_str());
}
}
+
+TSK_WALK_RET_ENUM
+content::file_act(TSK_FS_FILE * fs_file, TSK_OFF_T a_off, TSK_DADDR_T addr, char *buf,
+ size_t size, TSK_FS_BLOCK_FLAG_ENUM flags)
+{
+ if(opt_debug>1){
+ printf("file_act(fs_file=%p,addr=%"PRIuDADDR" buf=%p size=%d)\n",
+ fs_file,addr,buf,(int)size);
+ if(opt_debug>1 && segs.size()==0){
+ if(fwrite(buf,size,1,stdout)!=1) err(1,"fwrite");
+ printf("\n");
+ }
+ }
+
+ if(size==0) return TSK_WALK_CONT; // can't do much with this...
+
+ if(opt_no_data==false){
+ if (flags & TSK_FS_BLOCK_FLAG_SPARSE){
+ if (size < MAX_SPARSE_SIZE && !invalid) {
+ /* Manufacture NULLs that correspond with a sparse file */
+ char nulls[65536];
+ memset(nulls,0,sizeof(nulls));
+ for(size_t i=0; i<size; i += sizeof(nulls)){
+ size_t bytes_to_hash = sizeof(nulls);
+ if ( i + bytes_to_hash > size) bytes_to_hash = size - i;
+ add_bytes(nulls, a_off + i,bytes_to_hash);
+ }
+ } else {
+ set_invalid(true); // make this data set invalid
+ }
+ }
+ else {
+ add_bytes(buf,a_off,size); // add these bytes to the file
+ }
+ }
+
+ /* "Address 0 is reserved in ExtX and FFS to denote a "sparse"
+ block (one which is all zeros). TSK knows this and returns
+ zeros when a file refers to block 0. You can check the 'flags'
+ argument to the callback to determine if the data is from
+ sparse or compressed data. RAW means that the data in the
+ buffer was read from the disk.
+
+ TSK_FS_BLOCK_FLAG_RAW - data on the disk
+ TSK_FS_BLOCK_FLAG_SPARSE - a whole
+ TSK_FS_BLOCK_FLAG_COMP - the file is compressed
+ */
+
+ uint64_t fs_offset = addr * fs_file->fs_info->block_size;
+ uint64_t img_offset = current_partition_start + fs_offset;
+
+ if(opt_sector_hash){
+ if(h_sectorhash==0){
+ h_sectorhash = new md5_generator();
+ sectorhash_byte_counter = 0;
+ sectorhash_initial_offset = (int64_t)a_off;
+ }
+ h_sectorhash->update((const uint8_t *)buf,size);
+ sectorhash_byte_counter += size;
+ if (sectorhash_byte_counter==sectorhash_size){
+ add_seg(0,0,sectorhash_initial_offset,sectorhash_byte_counter,flags,h_sectorhash->final().hexdigest());
+ }
+ if (sectorhash_byte_counter>=sectorhash_size){
+ delete h_sectorhash;
+ h_sectorhash=0;
+ }
+ return TSK_WALK_CONT;
+ }
+
+ /* We are not sector hashing; try to determine disk runs */
+ if(segs.size()>0){
+ /* Does this next segment fit after the prevous segment logically? */
+ if(segs.back().next_file_offset()==(uint64_t)a_off){
+
+ /* if both the last and the current are sparse, this can be extended. */
+ if((segs.back().flags & TSK_FS_BLOCK_FLAG_SPARSE) &&
+ (flags & TSK_FS_BLOCK_FLAG_SPARSE)){
+
+ segs.back().len += size;
+ return TSK_WALK_CONT;
+ }
+
+ /* If both are compressed, then this can be extended? */
+ if((segs.back().flags & TSK_FS_BLOCK_FLAG_COMP) &&
+ (flags & TSK_FS_BLOCK_FLAG_COMP) &&
+ (segs.back().img_offset + segs.back().len == img_offset)){
+ segs.back().len += size;
+ return TSK_WALK_CONT;
+ }
+
+ /* See if we can extend the last segment in the segment list,
+ * or if this is the start of a new fragment.
+ */
+ if((segs.back().flags & TSK_FS_BLOCK_FLAG_RAW) &&
+ (flags & TSK_FS_BLOCK_FLAG_RAW) &&
+ (segs.back().img_offset + segs.back().len == img_offset)){
+ segs.back().len += size;
+ return TSK_WALK_CONT;
+ }
+ }
+ }
+ /* Need to add a new element to the list */
+ add_seg(img_offset,fs_offset,(int64_t)a_off,size,flags,"");
+ return TSK_WALK_CONT;
+}
diff --git a/tools/fiwalk/src/content.h b/tools/fiwalk/src/content.h
index d3c769824ad6aeb7e5ca23668b778f627fad8e9b..46b9a705c8df04d71edafd543a438687e7b86e63 100644
--- a/tools/fiwalk/src/content.h
+++ b/tools/fiwalk/src/content.h
@@ -8,12 +8,6 @@
#include <vector>
#include <string>
-
-//#ifndef HAVE_ERR_H
-//extern void err(int eval, const char *fmt, ...);
-//extern void errx(int eval, const char *fmt, ...);
-//#endif
-
/* Structure for keeping track of file segments */
class seg {
public:;
@@ -21,6 +15,7 @@ public:;
uint64_t img_offset; // offset from beginning of image
uint64_t file_offset; // logical number of bytes from beginning of file
uint64_t len; // number of bytes
+ std::string md5; // md5 if we are sector hashing, otherwise null
TSK_FS_BLOCK_FLAG_ENUM flags; //
uint64_t next_file_offset() {return file_offset + len;}
uint64_t next_img_offset() {return img_offset + len;}
@@ -53,8 +48,9 @@ class content {
md5_generator h_md5;
sha1_generator h_sha1;
- md5_generator h_sectorhash;
- uint64_t sectorhash_counter;
+ md5_generator *h_sectorhash;
+ uint64_t sectorhash_byte_counter; // number of bytes that have been hashed
+ uint64_t sectorhash_initial_offset;
seglist segs; // the segments that make up the file
uint64_t total_bytes;
std::vector<std::string> sectorhashes; // a vector of sector hashes, if any have been computed
@@ -65,7 +61,13 @@ class content {
fd_save(0),
fd_temp(0),
tempdir("/tmp"),
- sectorhash_counter(0),
+ tempfile_path(""),
+ h_md5(),
+ h_sha1(),
+ h_sectorhash(0),
+ sectorhash_byte_counter(0),
+ sectorhash_initial_offset(0),
+ segs(),
total_bytes(0) {
}
~content();
@@ -74,13 +76,16 @@ class content {
std::string filename() { return evidence_dirname + evidence_filename; }
std::string filemagic(); // returns output of the 'file' command or libmagic
void add_seg(int64_t img_offset,int64_t fs_offset,int64_t file_offset,
- int64_t len, TSK_FS_BLOCK_FLAG_ENUM flags);
+ int64_t len, TSK_FS_BLOCK_FLAG_ENUM flags,const std::string &hash);
void add_bytes(const u_char *buf,uint64_t file_offset,ssize_t size);
void add_bytes(const char *buf,uint64_t file_offset,ssize_t size){ // handle annoying sign problems
add_bytes((const u_char *)buf,file_offset,size);
}
void write_record(); // writes the ARFF record for this content
+ TSK_WALK_RET_ENUM file_act(TSK_FS_FILE * fs_file, TSK_OFF_T a_off, TSK_DADDR_T addr, char *buf,
+ size_t size, TSK_FS_BLOCK_FLAG_ENUM flags);
+
};
#endif
diff --git a/tools/fiwalk/src/fiwalk.cpp b/tools/fiwalk/src/fiwalk.cpp
index 92b6bb5969fe86b98a75804f4f0d5bab8e4743bd..9cfcf1c1793192e3b170462d546f7afd22b9f75b 100644
--- a/tools/fiwalk/src/fiwalk.cpp
+++ b/tools/fiwalk/src/fiwalk.cpp
@@ -78,6 +78,7 @@ bool opt_allocated_only = false;
bool opt_body_file = false;
bool opt_ignore_ntfs_system_files = false;
bool opt_parent_tracking = false;
+bool opt_sector_hash = false;
const char *config_file = 0;
int file_count_max = 0;
@@ -89,8 +90,6 @@ int opt_k = 4;
u_int sectorhash_size=512;
-//bool opt_compute_sector_hashes = false;
-//bool opt_print_sector_hashes = false;
namelist_t namelist; // names of files that we want to find
@@ -153,6 +152,7 @@ void usage()
printf("Ways to make this program run slower:\n");
printf(" -M = Report MD5 for each file (default on)\n");
printf(" -1 = Report SHA1 for each file (default on)\n");
+ printf(" -S nnnn = Perform sector hashes every nnnn bytes\n");
#ifdef HAVE_LIBMAGIC
printf(" -f = Enable LIBMAGIC (disabled by default)");
#else
@@ -497,7 +497,7 @@ int main(int argc, char * const *argv1)
argv = (TSK_TCHAR * const*) argv1;
#endif
- while ((ch = GETOPT(argc, argv, _TSK_T("A:a:C:dfG:gmv1IMX:T:VZn:c:b:xOzh?"))) > 0 ) { // s: removed
+ while ((ch = GETOPT(argc, argv, _TSK_T("A:a:C:dfG:gmv1IMX:S:T:VZn:c:b:xOzh?"))) > 0 ) { // s: removed
switch (ch) {
case _TSK_T('1'): opt_sha1 = true;break;
case _TSK_T('m'):
@@ -516,10 +516,6 @@ int main(int argc, char * const *argv1)
break;
case _TSK_T('C'): file_count_max = TATOI(OPTARG);break;
case _TSK_T('d'): opt_debug++; break;
-// case _TSK_T('E'):
-// opt_print_sector_hashes = true;
-// opt_compute_sector_hashes=true;
-// break;
case _TSK_T('f'): opt_magic = true;break;
case _TSK_T('g'): opt_no_data = true; break;
case _TSK_T('b'): opt_get_fragments = false; break;
@@ -528,7 +524,9 @@ int main(int argc, char * const *argv1)
case _TSK_T('I'): opt_ignore_ntfs_system_files=true;break;
case _TSK_T('M'): opt_md5 = true;
case _TSK_T('O'): opt_allocated_only=true; break;
-// case _TSK_T('S'): sectorhash_size = TATOI(OPTARG); break;
+ case _TSK_T('S'):
+ opt_sector_hash = true;
+ sectorhash_size = TATOI(OPTARG); break;
case _TSK_T('T'):
#ifdef TSK_WIN32
convert(OPTARG, &opt_arg);
diff --git a/tools/fiwalk/src/fiwalk.h b/tools/fiwalk/src/fiwalk.h
index 4ddd59399e5b831ddfee1d7d772a2b5482f63d68..488ca84897cc94eb3d17ac144f8ac512daf58b45 100644
--- a/tools/fiwalk/src/fiwalk.h
+++ b/tools/fiwalk/src/fiwalk.h
@@ -112,12 +112,12 @@ extern bool opt_md5; // do we need md5s?
extern bool opt_sha1; // do we need sha1s?
extern string save_outdir;
extern bool opt_get_fragments;
-//extern bool opt_compute_sector_hashes;
extern int opt_debug;
extern bool opt_no_data;
extern bool opt_allocated_only;
extern bool opt_body_file;
extern bool opt_ignore_ntfs_system_files;
+extern bool opt_sector_hash;
extern int current_partition_num;
extern int64_t current_partition_start;
extern const char *config_file;
@@ -140,8 +140,6 @@ void file_infot(const string name,time_t t0);
void file_infot(const string name,time_t t0, TSK_FS_TYPE_ENUM ftype);
extern u_int sectorhash_size; // for the computation of sector hashes
-//extern bool opt_compute_sector_hashes;
-//extern bool opt_print_sector_hashes;
extern namelist_t namelist; // names of files that we want to find
diff --git a/tools/fiwalk/src/fiwalk_tsk.cpp b/tools/fiwalk/src/fiwalk_tsk.cpp
index 26996a2f84b36f4ced64aa04b16beb570196276a..f75b72a30e8e4dbdbd6289c370b9b4cb08157267 100644
--- a/tools/fiwalk/src/fiwalk_tsk.cpp
+++ b/tools/fiwalk/src/fiwalk_tsk.cpp
@@ -67,88 +67,7 @@ file_act(TSK_FS_FILE * fs_file, TSK_OFF_T a_off, TSK_DADDR_T addr, char *buf,
size_t size, TSK_FS_BLOCK_FLAG_ENUM flags, void *ptr)
{
content *ci = (content *)ptr;
-
- if(opt_debug>1){
- printf("file_act(fs_file=%p,addr=%"PRIuDADDR" buf=%p size=%d)\n",
- fs_file,addr,buf,(int)size);
- if(opt_debug>1 && ci->segs.size()==0){
- if(fwrite(buf,size,1,stdout)!=1) err(1,"fwrite");
- printf("\n");
- }
- }
-
- if(size==0) return TSK_WALK_CONT; // can't do much with this...
-
- if(opt_no_data==false){
- if (flags & TSK_FS_BLOCK_FLAG_SPARSE){
- if (size < MAX_SPARSE_SIZE && !ci->invalid) {
- /* Manufacture NULLs that correspond with a sparse file */
- char nulls[65536];
- memset(nulls,0,sizeof(nulls));
- for(size_t i=0; i<size; i += sizeof(nulls)){
- size_t bytes_to_hash = sizeof(nulls);
- if ( i + bytes_to_hash > size) bytes_to_hash = size - i;
- ci->add_bytes(nulls, a_off + i,bytes_to_hash);
- }
- } else {
- ci->set_invalid(true); // make this data set invalid
- }
- }
- else {
- ci->add_bytes(buf,a_off,size); // add these bytes to the file
- }
- }
-
- /* "Address 0 is reserved in ExtX and FFS to denote a "sparse"
- block (one which is all zeros). TSK knows this and returns
- zeros when a file refers to block 0. You can check the 'flags'
- argument to the callback to determine if the data is from
- sparse or compressed data. RAW means that the data in the
- buffer was read from the disk.
-
- TSK_FS_BLOCK_FLAG_RAW - data on the disk
- TSK_FS_BLOCK_FLAG_SPARSE - a whole
- TSK_FS_BLOCK_FLAG_COMP - the file is compressed
- */
-
- uint64_t fs_offset = (addr)*fs_file->fs_info->block_size;
- uint64_t img_offset = current_partition_start + fs_offset;
-
- if(ci->segs.size()>0){
- /* Does this next segment fit after the prevous segment logically? */
- if(ci->segs.back().next_file_offset()==(uint64_t)a_off){
-
- /* if both the last and the current are sparse, this can be extended. */
- if((ci->segs.back().flags & TSK_FS_BLOCK_FLAG_SPARSE) &&
- (flags & TSK_FS_BLOCK_FLAG_SPARSE)){
-
- ci->segs.back().len += size;
- return TSK_WALK_CONT;
- }
-
-
- /* If both are compressed, then this can be extended? */
- if((ci->segs.back().flags & TSK_FS_BLOCK_FLAG_COMP) &&
- (flags & TSK_FS_BLOCK_FLAG_COMP) &&
- (ci->segs.back().img_offset + ci->segs.back().len == img_offset)){
- ci->segs.back().len += size;
- return TSK_WALK_CONT;
- }
-
- /* See if we can extend the last segment in the segment list,
- * or if this is the start of a new fragment.
- */
- if((ci->segs.back().flags & TSK_FS_BLOCK_FLAG_RAW) &&
- (flags & TSK_FS_BLOCK_FLAG_RAW) &&
- (ci->segs.back().img_offset + ci->segs.back().len == img_offset)){
- ci->segs.back().len += size;
- return TSK_WALK_CONT;
- }
- }
- }
- /* Need to add a new element to the list */
- ci->add_seg(img_offset,fs_offset,(int64_t)a_off,size,flags);
- return TSK_WALK_CONT;
+ return ci->file_act(fs_file,a_off,addr,buf,size,flags);
}
/* This is modeled on print_dent_act printit in ./tsk/fs/fls_lib.c
@@ -343,21 +262,7 @@ process_tsk_file(TSK_FS_FILE * fs_file, const char *path)
/* Processing for regular files: */
if(fs_file->name->type == TSK_FS_NAME_TYPE_REG){
-
if(ci.do_plugin && ci.total_bytes>0) plugin_process(ci.tempfile_path);
-
- /* Output the sector hashes to text or XML file if requested */
-// if(opt_compute_sector_hashes){
-// int count = 1;
-// for(vector<string>::const_iterator it = ci.sectorhashes.begin();
-// it!=ci.sectorhashes.end(); it++){
-// if(opt_print_sector_hashes){
-// if(t) fprintf(t,"sectorhash: %s %d\n",(*it).c_str(),count);
-// if(x) x->xmlout("sectorhash",*it,"",false);
-// }
-// count++;
-// }
-// }
}
/* END of file processing */
@@ -377,7 +282,7 @@ dir_act(TSK_FS_FILE * fs_file, const char *path, void *ptr)
{
/* Ignore NTFS System files */
if (opt_ignore_ntfs_system_files
- && (TSK_FS_TYPE_ISNTFS(fs_file->fs_info->ftype))
+ && (TSK_FS_TYPE_ISNTFS(fs_file->fs_info->ftype) || TSK_FS_TYPE_ISFAT(fs_file->fs_info->ftype))
&& (fs_file->name->name[0] == '$'))
return TSK_WALK_CONT;
@@ -568,7 +473,7 @@ void process_scalpel_audit_file(TSK_IMG_INFO *img_info,const char *audit_file)
file_info("carvelength",length);
}
- ci.add_seg(start,start,0,r2,TSK_FS_BLOCK_FLAG_RAW); // may not be able to read it all
+ ci.add_seg(start,start,0,r2,TSK_FS_BLOCK_FLAG_RAW,""); // may not be able to read it all
ci.add_bytes(buf2,0,r2);
ci.write_record();
free(buf2);
diff --git a/tools/fiwalk/src/hash_t.h b/tools/fiwalk/src/hash_t.h
index e18b1b2716e0b8ed2f28286932c57d2f6ae568e3..64e948e22e8b8a081bd97a1ef865a25bc70849be 100644
--- a/tools/fiwalk/src/hash_t.h
+++ b/tools/fiwalk/src/hash_t.h
@@ -270,24 +270,6 @@ class hash_generator__:T { /* generates the hash */
/** Compute a sha1 from a buffer and return the hash */
static hash__<T> hash_buf(const uint8_t *buf,size_t bufsize){
/* First time through find the SHA1 of 512 NULLs */
-#if 0
- if(sha1_ctr==0){
- uint8_t b2[512];
- EVP_MD_CTX ctx2;
- unsigned int len = sizeof(sha1_512nulls);
- memset(b2,0,sizeof(b2));
- EVP_MD_CTX_init(&ctx2);
- EVP_DigestInit_ex(&ctx2, EVP_sha1(), NULL);
- EVP_DigestUpdate(&ctx2,b2,sizeof(b2));
- EVP_DigestFinal(&ctx2,sha1_512nulls,&len);
- sha1_ctr++;
- }
-
- /* If the input is 512 bytes long and all NULLs, use our SHA1 of 512 NULLs */
- if(bufsize==512 && iszero(buf,bufsize)){
- return sha1_t(sha1_512nulls);
- }
-#endif
hash_generator__ g;
g.update(buf,bufsize);
return g.final();