modules/search_files/search_files.module

<?php
// $Id: search_files.module,v 1.31 2009/06/22 21:45:04 thl Exp $

/**
 * @file
 * Used to index all files in directory(s) on the server
 */
 
 
/**
 * Implementation of hook_menu()
 *
 * @return $items = array of menu items
 */
function search_files_menu() {
  $items['admin/settings/search_files'] = array( 
    'title' => t('Search Files'),
    'description' => t('Manage files search'),
    'page callback' => 'search_files_dashboard',
    'access arguments' => array('administer search_files configuration'),
    'type' => MENU_NORMAL_ITEM,
  );
  $items['admin/settings/search_files/dashboard'] = array( 
    'title' => t('Dashboard'),
    'description' => t('Dashboard for Search Files Module'),
    'page callback' => 'search_files_dashboard',
    'access arguments' => array('administer search_files configuration'),
    'type' => MENU_DEFAULT_LOCAL_TASK,
  );
  $items['admin/settings/search_files/settings'] = array( 
    'title' => t('Settings'),
    'description' => t('Change settings for Search Files Module'),
    'page callback' => 'drupal_get_form',
    'page arguments' => array('search_files_settings'),
    'access arguments' => array('administer search_files configuration'),
    'type' => MENU_LOCAL_TASK,
  );
  $items['admin/settings/search_files/helpers'] = array( 
    'title' => t('Helpers'),
    'description' => t('List Helper Apps for Search Files'),
    'page callback' => 'search_files_helper_list',
    'access arguments' => array('administer search_files configuration'),
    'type' => MENU_LOCAL_TASK,
  );
  $items['admin/settings/search_files/helpers/list'] = array( 
    'title' => t('List'),
    'description' => t('List Helper Apps for Search Files'),
    'page callback' => 'search_files_helper_list',
    'access arguments' => array('administer search_files configuration'),
    'type' => MENU_DEFAULT_LOCAL_TASK,
  );
  $items['admin/settings/search_files/helpers/add'] = array( 
    'title' => t('Add'),
    'description' => t('Add Helper Apps for Search Files'),
    'page callback' => 'drupal_get_form',
    'page arguments' => array('search_files_helper_add'),
    'access arguments' => array('administer search_files configuration'),
    'type' => MENU_LOCAL_TASK,
  );
  $items['admin/settings/search_files/directories'] = array( 
    'title' => t('Directories'),
    'description' => t('list directories that will be searched'),
    'page callback' => 'search_files_directories_list',
    'access arguments' => array('administer search_files configuration'),
    'type' => MENU_LOCAL_TASK,
  );
  $items['admin/settings/search_files/directories/list'] = array( 
    'title' => t('List'),
    'description' => t('list directories that will be searched'),
    'page callback' => 'search_files_directories_list',
    'access arguments' => array('administer search_files configuration'),
    'type' => MENU_DEFAULT_LOCAL_TASK,
  );
   $items['admin/settings/search_files/directories/add'] = array( 
    'title' => t('Add'),
    'description' => t('Add a directory that will be searched'),
    'page callback' => 'drupal_get_form',
    'page arguments' => array('search_files_directories_add'),
    'access arguments' => array('administer search_files configuration'),
    'type' => MENU_LOCAL_TASK,
  );
  $items['admin/settings/search_files/directory/delete'] = array( 
    'title' => t('Delete Directory'),
    'description' => t('Delete Searchable Driectory'),
    'page callback' => 'search_files_directory_confirm_delete',
    'access arguments' => array('administer search_files configuration'),
    'type' => MENU_CALLBACK,
  );
  $items['admin/settings/search_files/directory/edit'] = array( 
    'title' => t('Edit Directory'),
    'description' => t('Edit directory path'),
    'page callback' => 'drupal_get_form',
    'page arguments' => array('search_files_directory_edit'),
    'access arguments' => array('administer search_files configuration'),
    'type' => MENU_CALLBACK,
  );
  $items['admin/settings/search_files/helpers/edit'] = array( 
    'title' => t('Edit Helper'),
    'description' => t('Edit a Helper App'),
    'page callback' => 'drupal_get_form',
    'page arguments' => array('search_files_helper_edit'),
    'access arguments' => array('administer search_files configuration'),
    'type' => MENU_CALLBACK,
  );
  $items['admin/settings/search_files/helpers/delete'] = array( 
    'title' => t('Delete Helper'),
    'description' => t('Delete a Helper App'),
    'page callback' => 'search_files_helper_confirm_delete',
    'access arguments' => array('administer search_files configuration'),
    'type' => MENU_CALLBACK,
  );
    $items['admin/settings/search_files/update_index'] = array( 
    'title' => t('Update the index'),
    'description' => t('manually run hook_upate_index'),
    'page callback' => 'search_files_update_index',
    'access arguments' => array('administer search_files configuration'),
    'type' => MENU_CALLBACK,
  );
  return $items;
}
/**
 * get an array of helper programs
 *
 * @return $helpers = array($helper->extension => $helper->helper_path);
 */
function search_files_get_helpers() {
  // Get all the registered helper applications and put them in static variable to elimiate unnecessary db queries
  // in search_files_nodeapi(). The query log feature of the dev module pointed out that this query was done
  // many times instead of once. Making $helpers a static variable reduced the number of queries by 25%.
  static $helpers = array();
  $result = db_query('SELECT * FROM {search_files_helpers}'); 
  while ($helper = db_fetch_object($result)) { 
    $helpers[$helper->extension] = $helper->helper_path;
  } 
  return $helpers;
}
/**
 * check to make sure the directory the user wants to delete is a 
 * valid directory id number, then call the function to generate 
 * the confirmation form 
 *
 * @return $output = html of the form
 */
function search_files_directory_confirm_delete() {
  $directory_id = arg(5)  ;
  if ($directory_id > 0 && is_numeric($directory_id)) {
    return drupal_get_form('search_files_directory_confirm_delete_form', $directory_id);
  }
}
/**
 * get the confirmation form to confirm deletion of a directory 
 * from the search_files_directories table
 *
 * @param (array) $form_state
 * @param (int) $directory_id
 * @return $output = html of the form
 */
function search_files_directory_confirm_delete_form(&$form_state, $directory_id) {
  $directory_path = db_result(db_query("SELECT directory FROM {search_files_directories} WHERE id = '%s'", $directory_id)); 
  $form = array();
  $form['search_files_directory_id'] = array(
    '#type' => 'hidden', 
    '#value' => $directory_id
  );
  $form['search_files_directory_path'] = array(
    '#type' => 'hidden', 
    '#value' => $directory_path,
  );
  return confirm_form($form,
    t('Are you sure you want to delete the %directory directory? The text extracted from this directory be removed from the search index.', array('%directory' => $directory_path)), 
    'admin/settings/search_files/directories/list',
    t('This action cannot be undone.'),
    t('Delete'),
    t('Cancel'));
}
/**
 * deletes the directory from the search_files_directories table after 
 * confirmation from the user, also deletes the files from the 
 * search_files_files table and removes the data from the search_dataset 
 * table
 *
 * @param unknown_type $form_id
 * @param unknown_type $form_values
 */
function search_files_directory_confirm_delete_form_submit($form_id, $form_values) {
  //drupal_set_message('form_values = <pre>'.print_r($form_values, true).'</pre>');
  $directory_path = $form_values['values']['search_files_directory_path'];
  search_files_delete_content_by_directory_fullpath($directory_path);
  drupal_set_message(t('Directory %directory removed from search index', array('%directory' => $directory_path)));
  drupal_goto('admin/settings/search_files/directories/list');
}

function search_files_delete_content_by_directory_fullpath($full_path) {
  $sql = "DELETE s.*, f.*, d.*
          FROM {search_dataset} AS s
          INNER JOIN {search_files_files} AS f ON s.`sid` = f.`id`
          INNER JOIN {search_files_directories} AS d ON f.`directory_id` = d.`id`
          WHERE (     s.`type` = 'search_files' 
                  AND d.`directory` = '%s'
                )";
  db_query($sql, $full_path);
}

function search_files_delete_content_by_file_fullpath($full_path) {
  $sql = "DELETE s.*, f.*
          FROM {search_dataset} AS s
          INNER JOIN {search_files_files} AS f ON s.`sid` = f.`id`
          WHERE (     s.`type` = 'search_files' 
                  AND f.`full_path` = '%s'
                )";
  db_query($sql, $full_path);
}

/**
 * Check whether we run in PHP safe_mode
 */
function search_files_issafemode() {
  return preg_match('/(1|on)/i', @ini_get("safe_mode"));
}

/**
 * generates the directory edit form, it does this by grabbing the 
 * directory add form and populates the #default_value fields for 
 * the directory in question
 *
 * @return (array) $form
 */
function search_files_directory_edit() {
  $sql = "
    SELECT
      *
    FROM
      {search_files_directories}
    WHERE
      ID = '%s'
  ";
  $result = db_fetch_object(db_query($sql, arg(5)));
  $form = array();

  $form = search_files_directories_add();
  $form['search_files_directory']['#default_value'] = $result->directory;
  $form['search_files_id'] = array(
    '#type' => 'value',
    '#value' => $result->id,
  );
  return $form;
}

/**
 * Updates the directory row in the serach_files_directories table from 
 * data given by the search_files_directory_edit form
 *
 * @param (array) $form_id
 * @param (array) $form_values
 */
function search_files_directory_edit_submit($form_id, $form_values) {
  //drupal_set_message('form_values= <pre>'.print_r($form_values, true).'</pre>');
  $sql = "
    UPDATE
      {search_files_directories}
    SET
      `directory`='%s'
    WHERE
      `id`='%s'
  ";
  $result = db_query($sql, $form_values['values']['search_files_directory'], $form_values['values']['search_files_id']);
  if ($result) {
    drupal_set_message(t('Directory %directory has been updated', array('%directory' => $form_values['values']['search_files_directory'])));
    drupal_goto('admin/settings/search_files/directories/list');
  }
}

/**
 * gets a list of directories to inde from the search_files_directories 
 * table and themes the list in a table
 *
 * @return (string) $output = themed table of directories
 */
function search_files_directories_list() {
  $result = db_query('SELECT * FROM {search_files_directories}');
  $directories[] = array();
  $destination = drupal_get_destination();
  while ($directory = db_fetch_object($result)) {
    $directories[] = array( $directory->directory, l(t('Edit'), 'admin/settings/search_files/directory/edit/'. $directory->id), l(t('Delete'), 'admin/settings/search_files/directory/delete/'. $directory->id), array($destination));
  } 
  $header = array( 'Directory', array('data' => t('Operations'), 'colspan' => '3'));
  $output .= theme('table', $header, $directories);
  return $output;
}

/**
 * generates the form to add a directory to search_files_directories table
 *
 * @return (array) $form
 */
function search_files_directories_add() {
  $form = array();
  $form['instructions']= array(
    '#type' => 'markup',
    '#value' => t('Make sure the directory is readable by the web server, i.e. apache.'),
  );
  $form['search_files_directory'] = array(
    '#type' => 'textfield',
    '#title' => t('Directory Path'),
    '#size' => 80,
    '#maxlength' => 150,
    '#required' => FALSE,
    '#description' => t('The path of the directory to search. Relative pathes are based on %directory.', array('%directory' => getcwd())),
  );
  $form['submit'] = array(
    '#type' => 'submit',
    '#value' => 'Submit',
  );
  return $form;
}

/**
 * adds a row to the search_files directories table with  the 
 * information provided by the search_files_directory_add_form
 *
 * @param unknown_type $form_id
 * @param unknown_type $form_values
 */
function search_files_directories_add_submit($form_id, $form_values) {
  //drupal_set_message('$form_values = <pre>'.print_r($form_values, true).'</pre>');
  $sql = "
    INSERT INTO
      {search_files_directories}
    SET
      `directory`='%s'
  ";
  $directory = $form_values['values']['search_files_directory'];
  if (!preg_match('/^\//', $directory)) {
    $directory = sprintf("%s/%s", getcwd(), $directory);
  }
  $directory = preg_replace('/\/+$/', '', $directory);
  $result = db_query($sql, $directory, $form_values['values']['search_files_label']);
  if ($result) {
    drupal_set_message(t('Directory %directory has been added.', array('%directory' => $form_values['values']['search_files_directory'])));
    drupal_goto('admin/settings/search_files/directories/list');
  }
}

/**
 * check to make sure the $helper_id is valid (a number greater 
 * than zero) then it fetches the deletion confirmation form, 
 * then returns it 
 *
 * @return (array) $form
 */
function search_files_helper_confirm_delete() {
  $helper_id = arg(5)  ;
  if ($helper_id > 0 && is_numeric($helper_id)) {
    return drupal_get_form('search_files_helper_confirm_delete_form', $helper_id);
  }
}

/**
 * generate the deletion confirmation form for helper apps and return the form
 *
 * @param unknown_type $form_state
 * @param (int) $helper_id
 * @return (array) $form
 */
function search_files_helper_confirm_delete_form(&$form_state, $helper_id) {
  $helper_name = db_result(db_query("SELECT name FROM {search_files_helpers} WHERE id = '%s'", $helper_id)); 
  $form = array();
  $form['search_files_helper_id'] = array(
    '#type' => 'hidden', 
    '#value' => $helper_id
  );
  $form['search_files_helper_name'] = array(
    '#type' => 'hidden', 
    '#value' => $helper_name,
  );
  return confirm_form($form,
    t('Are you sure you want to delete the %name helper? The text extracted by this helper will remain in the search index until the directory is reindexed.', array('%name' => $helper_name)), 
    'admin/settings/search_files/helpers/list',
    t('This action cannot be undone.'),
    t('Delete'),
    t('Cancel'));
}

/**
 * remove row for the helper app from the search_files_helpers table
 *
 * @param unknown_type $form_id
 * @param (array) $form_values
 */
function search_files_helper_confirm_delete_form_submit($form_id, $form_values) {
  //drupal_set_message('form_values = <pre>'.print_r($form_values, true).'</pre>');
  $sql = "
    DELETE FROM
      {search_files_helpers}
    WHERE
      `id`='%s'
  ";
  $result = db_query($sql, $form_values['values']['search_files_helper_id']);
  drupal_goto('admin/settings/search_files/helpers/list');
}
/**
 * display a themes table of the current helper apps set up in the system
 *
 * @return (string) html table
 */
function search_files_helper_list() {
  $sql = "
    SELECT
      *
    FROM
     {search_files_helpers}
    ORDER BY
      `extension`
  ";
  $result = db_query($sql);
  $helpers[] = array();
  $destination = drupal_get_destination();
  while ($helper = db_fetch_object($result)) {
    $helpers[] = array($helper->name, $helper->extension, l(t('Edit'), 'admin/settings/search_files/helpers/edit/'. $helper->id), l(t('Delete'), 'admin/settings/search_files/helpers/delete/'. $helper->id), array($destination));
  } 
  $header = array(t('Helper name'), t('Extension'), array('data' => t('Operations'), 'colspan' => '3'));
  $output .= theme('table', $header, $helpers);
  return $output;
}

/**
 * Implementation of hook_perm()
 *
 * @return (array) permissions
 */
function search_files_perm() {
    return array('administer search_files configuration', 'view search_files results');
}

/**
 * generate form to add a helper app to the system
 *
 * @return (array) $form
 */
function search_files_helper_add() {
  $form['instructions'] = array('#type' => 'markup', '#value' => $instruction_text);	
  $form['search_files_name'] = array(
    '#type' => 'textfield',
    '#title' => t('Helper name'),
    '#size' => 50,
    '#maxlength' => 50,
    '#required' => TRUE,
    '#description' =>  t('A name for this helper configuration.'),
  );
  $form['search_files_extension'] = array(
    '#type' => 'textfield',
    '#title' => t('Extension'),
    '#size' => 10,
    '#maxlength' => 10,
    '#required' => TRUE,
    '#description' =>  t('Enter the extension for the files that you want the helper application to process. Do not include the period.'),
  );
  $form['search_files_helper_path'] = array(
    '#type' => 'textfield',
    '#title' => t('Helper path'),
    '#size' => 100,
    '#maxlength' => 100,
    '#default_value' => '/path/to/helper/command %file%',
    '#validate' => array('search_files_helpers_validate_add_edit' => array()),
    '#required' => TRUE,
    '#description' => t('Enter the path to the helper application installed on your server. "%file%" is a placeholder for the path of the attachment file and is required. Include any command-line parameters as well (for example, pdftotext requires a - after the file to be processed).'),
  );
  $form['submit_done'] = array( 
    '#type' => 'submit', 
    '#value' => 'Save and Done',
  ); 
  $form['submit'] = array( 
    '#type' => 'submit', 
    '#value' => 'Save add another',
  ); 
  return $form; 
}

/**
 * validate the existance of the helper app supplied by the 
 * helper_add or helper_edit form
 *
 * @param unknown_type $field
 */
function search_files_helpers_validate_add_edit($field) {
  if (!preg_match('/%file%/', $field['#value'])) {
    form_set_error($field['#title'], t('"%field" must contain the token %file%', array('%field' => $field['#title'])));
  }

  // Check to see if helper app can be found
  $helper_file = preg_replace('/\s.+$/', '', $field['#value']);
  if (!file_exists($helper_file)) {
    form_set_error($field['#title'], t("Can't find helper app %helper -- please verify it is installed.", array('%helper' => $helper_file)));
  }
}

/**
 * fetch the helper add form, alter the form for use as an edit form, 
 * populate the #default_values fields and return the form
 *
 * @return unknown
 */
function search_files_helper_edit() {
  $sql = "
    SELECT
      *
    FROM
      {search_files_helpers}
    WHERE
      ID = '%s'
  ";
  $result = db_fetch_object(db_query($sql, arg(5)));
  $form = search_files_helper_add();
  $form['search_file_id'] = array(
    '#type' => 'value',
    '#value' => $result->id,
  );
  $form['search_files_helper_path']['#value'] = $result->helper_path;
  $form['search_files_extension']['#value'] = $result->extension;
  $form['search_files_name']['#value']= $result->name;
  unset($form['submit_done']);
  $form['submit']['#value'] = 'Update';
  
  
  return $form; 
}

/**
 * update the row in the table search_files_helpers for the given helper app
 *
 * @param unknown_type $form_id
 * @param unknown_type $form_values
 */
function search_files_helper_edit_submit($form_id, $form_values) {
  //drupal_set_message('form_values = <pre>'.print_r($form_values, true).'</pre>');
  $sql = "
    UPDATE
      {search_files_helpers}
    SET
      `name`='%s',
      `extension`='%s',
      `helper_path`='%s'
    WHERE
      `id`='%s'
  ";
  $result = db_query($sql, $form_values['clicked_button']['#post']['search_files_name'], $form_values['clicked_button']['#post']['search_files_extension'], $form_values['clicked_button']['#post']['search_files_helper_path'], $form_values['values']['search_file_id']);
  if ($result) {
    drupal_set_message(t('Helper app %helper_name has been updated', array('%helper_name' => $form_values['values']['search_files_name'])));
    drupal_goto('admin/settings/search_files/helpers/list');
  }
}

/**
 * insert a row in the search_files_helpers table for the helper 
 * app given by the form
 *
 * @param unknown_type $form_id
 * @param unknown_type $form_values
 */
function search_files_helper_add_submit($form_id, $form_values) {
  //drupal_set_message('form_values = <pre>'.print_r($form_values, true).'</pre>');
  $sql = "
    INSERT INTO
     {search_files_helpers}
   SET
    `name`='%s',
    `extension`='%s',
    `helper_path`='%s'
  ";
  $result = db_query($sql, $form_values['values']['search_files_name'], $form_values['values']['search_files_extension'], $form_values['values']['search_files_helper_path']);
  if ($result) {
    drupal_set_message(t('%helper helper added', array('%helper' => $form_values['values']['search_files_name'])));
  }
  if ($form_values['clicked_button']['#id'] == 'edit-submit-done') {
    drupal_goto('admin/settings/search_files/helpers/list');
  }
}

/**
 * generate the dashboard page
 */
function search_files_dashboard() {
  $output = '';
  $lastindex = variable_get('search_files_last_index', '0');
  $output .= sprintf("%s = %s UTC<br>\n", t('Last Index'),
                     format_date($lastindex, $type = 'custom', $format = 'Y-m-d H:i:s', $timezone = NULL, $langcode = NULL));

  $directoryrescanage = variable_get('search_files_directoryrescanage', 'unset');
  $output .= sprintf("%s = %s [sec]<br>\n", t('Directory Rescan Age'), $directoryrescanage);

  $nextdirectoryrescan = $lastindex + $directoryrescanage;
  $output .= sprintf("%s = %s UTC<br>\n", t('Next Directory (Re-)Scan at or after'),
                     format_date($nextdirectoryrescan, $type = 'custom', $format = 'Y-m-d H:i:s', $timezone = NULL, $langcode = NULL));

  $sql = "SELECT count(*) FROM {search_files_directories};";
  $result = db_query($sql);
  $result = db_result($result);
  $output .= sprintf("Number of Directories configured = %s<br>\n", $result);

  $sql = "SELECT count(*) FROM {search_files_files};";
  $result = db_query($sql);
  $result = db_result($result);
  $output .= sprintf("Files found in configured Directories and Subdirectories = %s<br>\n", $result);

  $sql = "SELECT count(*) FROM {search_files_files} WHERE ( `index_attempts` = 0 );";
  $result = db_query($sql);
  $result = db_result($result);
  $output .= sprintf("Files without index attempt = %s<br>\n", $result);

  $sql = "SELECT count(*) from {search_dataset} WHERE ( `type` = 'search_files' );";
  $result = db_query($sql);
  $result = db_result($result);
  $output .= sprintf("Files indexed = %s<br>\n", $result);

  $sql = "SELECT count(*) from {search_dataset} WHERE ( `type` = 'search_files' AND `reindex` > 0 );";
  $result = db_query($sql);
  $result = db_result($result);
  $output .= sprintf("Files indexed and scheduled for reindexing = %s<br>\n", $result);

  $sql = "SELECT count(*) FROM {search_files_helpers};";
  $result = db_query($sql);
  $result = db_result($result);
  $output .= sprintf("Number of Helpers configured = %s<br>\n", $result);

  // safe_mode will inhibit shell_exec()
  if (search_files_issafemode()) {
    $output .= t('WARNING! This server has safe_mode enabled, which inhibits use of helper applications');
  }
  else {
    $output .= t('Good. This server has safe_mode disabled, which allows use of helper applications');
  }

  return $output;
}

/**
 * generate the settings form for the search_files module using the 
 * system_settings_form() function
 *
 * @return unknown
 */
function search_files_settings() {
  $form = array();
  $form['search_files_label'] = array(
    '#title' => 'Search Label',
    '#type' => 'textfield',
    '#description' => 'What do you want the Search tab to be labeled?',
    '#default_value' => variable_get('search_files_label', 'Server Files'),
  );
  $form['search_files_directoryrescanage'] = array(
    '#title' => t('Directory Rescan Age'),
    '#type' => 'textfield',
    '#description' => 'Minimum time to wait before directories are (re)scanned for new files.',
    '#default_value' => search_files_variable_get_directoryrescanage(),
    '#field_suffix' => t('[sec]'),
  );
  return system_settings_form($form);
}

/*
 * Handle configuration setting and provide global default
 */
function search_files_variable_get_directoryrescanage()
{
  return variable_get('search_files_directoryrescanage', '86400');
}

/**
 * Implementation of hook_update_index()
 * 
 * lists all the files in the director(y/ies) and puts the files 
 * into the "search_files_files" table
 *
 * then indexes X(configurable) number of these files
 */
function search_files_update_index() {
  $helpers = search_files_get_helpers();
  // only update the list of files in the directories once per day
  if (variable_get('search_files_last_index', 0) < (time() - search_files_variable_get_directoryrescanage())) {
    variable_set('search_files_last_index', time());

    // hunt configured directories for new files and add them to the database
    $result = db_query('SELECT * FROM {search_files_directories}');
    while ($directory = db_fetch_object($result)) {
      search_files_list_directory($directory->directory, $directory->id);
    }

    // compare database to filesystem and remove vanished files from database
    $vanished = 0;
    $result = db_query('SELECT full_path FROM {search_files_files}');
    while ($file = db_fetch_object($result)) {
      $full_path = $file->full_path;
      if(!file_exists($full_path)) {
          $vanished++;
          search_files_delete_content_by_file_fullpath($full_path);
      }
    }
    if ($vanished > 0) {
        watchdog('Search Files', t('removed %vanished vanished files from index', array('%vanished' => $vanished)), array(), WATCHDOG_NOTICE);
    }
  }

  // premature end, safe_mode will inhibit shell_exec()
  if (search_files_issafemode()) {
    return;
  }
  
  $index_number = (int)variable_get('search_cron_limit', 100);
  $sql = "
    SELECT 
      *
    FROM 
      {search_files_files}
    LEFT JOIN 
    (
      SELECT 
        *
      FROM 
        {search_dataset}
      WHERE 
        `type` = 'search_files'
    ) AS `dataset` ON {search_files_files}.`id` = `dataset`.`sid` 
    WHERE
    (
      `dataset`.`reindex` IS NULL OR
      `dataset`.`reindex` != 0
    ) AND {search_files_files}.`index_attempts` <= 5
    LIMIT %s
  ";
   
  $result = db_query($sql, $index_number);
  
  while ($file = db_fetch_object($result)) {
    $full_path = $file->full_path;
    $file_name = explode('/', $full_path);
    $file_name = $file_name[count($file_name)-1];
    $file_extension = explode('.', $file_name);
    $file_extension = $file_extension[count($file_extension)-1];

    if (in_array($file_extension, array_keys($helpers))) {
      // record that we are attempting to index the file in case it hangs
      $increment_sql = "
        UPDATE
          {search_files_files}
        SET
          `index_attempts` = `index_attempts` + 1
        WHERE
          `id` = '%s'
      ";
      $increment_result = db_query($increment_sql, $file->id);
      if ($file->index_attempts >= 5) {
        // indexind failed too many times, record this to the log and continue
        watchdog('Search Files', t('failed to index %full_path after %attempts attempts', array('%full_path' => $file->full_path, '%attempts' => $file->index_attempts)), array(), WATCHDOG_ERROR);
        continue;
      }

      // skip over files which vanished before being indexed - cleanup is left to search_files_update_index()
      if(!file_exists($full_path)) {
        continue;
      }

      // %file% is a token that is placed in the helper's parameter list to represent the file path to the attachment.
      // We need to put the filename in quotes in case it contains spaces.
      $quoted_file_path = '"'. escapeshellcmd($full_path) .'"';
      $helper_command = preg_replace('/%file%/', $quoted_file_path, $helpers[$file_extension]);
      
      $file_text = shell_exec($helper_command);
      $file_text = search_files_convert_to_utf8($file_text);
      search_index($file->id, 'search_files', 'file name: '. $quoted_file_path .', text: '. $file_text); 
    }
    else{
      search_index($file->id, 'search_files', ''); 
    }
  }
}

/**
 * Get the host system's character encoding and convert text from it to UTF-8,
 * Drupal's default HTTP and database character encoding.
 */
function search_files_convert_to_utf8($text) {
  $encoding = iconv_get_encoding("output_encoding");
  $text = drupal_convert_to_utf8($text, $encoding);
  return $text;
}

/**
 * search_files_list_directory($directory, $id) will be called recursively
 * to traverse the directory tree and list all the files in the given 
 * directory, it will take the files it find and put them into the 
 * search_files_files table
 *
 * @param (string) $directory
 * @param (int) $id
 */
function search_files_list_directory($directory, $id) {
  if (!is_dir($directory)) {
    return;
  }
  watchdog('Search Files', 'Starting to list files in %directory', array('%directory' => $directory));
  $file_count = 0;
  $dir_count = 0;
  if ($dir = opendir($directory)) {
    while ($file = readdir($dir)) {
      $type = filetype($directory .'/'. $file);
      
      $full_path = escapeshellcmd(search_files_convert_to_utf8($directory .'/'. $file));
      if ($type == 'dir') {
        
        // make sure we don't retreverse the current or parent directory
        if (($file != '.') && ($file != '..')) {
          $dir_count++;
          search_files_list_directory($directory .'/'. $file, $id);
        }
      }
      else if ($type == 'file') {
        // Check to see if the file is already in the table, 
        $sql = "
          SELECT
            `id`
          FROM
            {search_files_files}
          WHERE
            `full_path`='%s'
        ";
        $result = db_result(db_query($sql, $full_path));
        // If the file is not in the table, insert it
        if (!$result) {
          $file_count++;
          $insert_sql = "
            INSERT INTO
              {search_files_files}
            SET
              `full_path`=\"%s\",
              `directory_id`='%s'
          ";
          // MySQL keeps throwing errors because of some file names that it doesn't like, this needs fixed
          $insert_result = db_query($insert_sql, $full_path, $id);
        }
      }
    }
  }
  watchdog('Search Files', 'Finished Listing files in %directory, found %file_count new file(s)', array('%directory' => $directory, '%file_count' => $file_count));
}

/**
 * implementation of hook_search()
 *
 * @param (string) $op
 * @param (string) $keywords
 * @return (mixed)
 */
function search_files_search($op = 'search', $keywords = null) {
  switch ($op) {
    case 'name':
      return variable_get('search_files_label', 'Server Files');
    case 'reset':
      $sql="
        UPDATE
          {search_dataset}
        SET
          `reindex`='%s'
        WHERE
          `type`='search_files'
      ";
      $result = db_query($sql, time());
      $sql = "UPDATE {search_files_files} SET `index_attempts`='0' WHERE 1";
      db_query($sql);
      break;
    case 'status':
      $return = array();
      $sql = "SELECT COUNT(*) as `count` FROM {search_files_files}";
      $return['total'] = db_result(db_query($sql));
      $sql = "
        SELECT 
          COUNT(*) AS `count`
        FROM 
          {search_files_files}
        LEFT JOIN 
        (
          SELECT 
            *
          FROM 
            {search_dataset}
          WHERE 
            `type` = 'search_files'
        ) AS `dataset` ON {search_files_files}.`id` = `dataset`.`sid` 
        WHERE
        (
          `dataset`.`reindex` IS NULL OR
          `dataset`.`reindex` != 0
        )
      ";
      $return['remaining'] = db_result(db_query($sql));
      return $return;
      break;
    case 'search':
    //drupal_set_message('$keywords = '.$keywords);  
      if (!user_access('view search_files results')) {
        return null;
      }
      $doc_root = $_SERVER['DOCUMENT_ROOT'];
      //drupal_set_message($doc_root);
      $file_extensions = search_files_get_file_extensions();
      $results = array();
      $find = do_search($keywords, 'search_files');
      //drupal_set_message('find = <pre>'.var_export($find, true).'</pre>');
      date_default_timezone_set('GMT');
      foreach ($find as $item) {
        //drupal_set_message('item = <pre>'. var_export($item, true).'</pre>');
        $sql = "
          SELECT
            *
          FROM
            {search_files_files}
          WHERE
            `id`='%s'
        ";
        $result = db_fetch_object(db_query($sql, $item->sid));
        $search_results = db_query("SELECT * FROM {search_dataset} s WHERE sid = %d AND type = '%s'", $item->sid, 'search_files');
        if ($dataset = db_fetch_object($search_results)) {
          //drupal_set_message('result = <pre>'.print_r($result, true).'</pre>');
          $file_name = explode('/', $result->full_path);
          $file_name = $file_name[count($file_name)-1];
          $file_extension = explode('.', $file_name);
          $file_extension = $file_extension[count($file_extension)-1];
          $link = str_replace($doc_root, '', $result->full_path);
          // we have had a problem where some links were returned relative to the search page,
          // not the document root, basically there was no beginning forward slash '/' in the
          // path
          if (strpos($link, '/') !== 0) {
            $link = '/'. $link;
          }
          // skip over files which vanished after being indexed - cleanup is left to search_files_update_index()
          if(file_exists($result->full_path)) {
            $results[] = array(
              'link' => $link,
              'date' => filectime($result->full_path),
              'type' => $file_extensions[$file_extension] .' file',
              'title' => $file_name .' ('. format_size(filesize($result->full_path)) .')',
              'snippet' => search_excerpt($keywords, $dataset->data),
            );
          }
        }
        date_default_timezone_set('MST');
      }
      //drupal_set_message('results = <pre>'.var_export($results, true).'</pre>');
      return $results;
      break;
  }
}

/**
 * generate the results page
 *
 * @param (array) $results
 * @param (string) $type
 * @return (string) $output
 */
function search_files_search_page($results, $type = 'search_files') {
  
  $output = '<dl class="search-results">';
  foreach ($results as $entry) {
    $output .= search_files_search_item_format($entry, $type);
  }
  $output .= '</dl>';
  $output .= theme('pager', NULL, 20, 0);
  return $output;
}

/**
 * format the result items before that are displayed
 *
 * @param (array) $item
 * @param (string) $type
 * @return (string) $output formatted string
 */
function search_files_search_item_format($item, $type) {
  //drupal_set_message('entry = <pre>'.var_export($item, true).'</pre>');
  $output = ' <dt class="title"><a href="'. check_url($item['link']) .'">'. check_plain($item['title']) .'</a></dt>';
  $info = array();
  if ($item['type']) {
    $info[] = check_plain($item['type']);
  }
  if ($item['user']) {
    // Add this here so the user name appears at the end of the output
    $item['extra'][] = $item['user'];
  }
  if ($item['date']) {
    $info[] = format_date($item['date'], 'small');
  }
  if (is_array($item['extra'])) {
    $info = array_merge($info, $item['extra']);
  }
  $output .= ' <dd>'. ($item['snippet'] ? '<p>'. $item['snippet'] .'</p>' : '') .'<p class="search-info">'. implode(' - ', $info) .'</p></dd>';

  return $output;
}

/**
 * returns an array of the file extensions with the extension as 
 * the key and the name of the file type as the value, this data 
 * is retrieved from the search_files_helpers table
 *
 * @return (array)
 */
function search_files_get_file_extensions() {
  $extensions = array();
  $result = db_query('SELECT extension, name FROM {search_files_helpers}'); 
  while ($helper = db_fetch_object($result)) { 
    $extensions[$helper->extension] = $helper->name;
  } 
  return $extensions;
}