modules/import_html/import_html_process.inc

<?php
/**
 * @file Actual routines for importing files.
 * 
 * 
 * @ingroup import_html Import HTML
 * @author Dan Morrison http://coders.co.nz/
 * @version $Id: import_html_process.inc,v 1.5.2.2 2009/03/14 03:18:51 dman Exp $
 *
 */


/**
 * Files have been selected, set them up for processing
 * 
 * @param $file_list an array of simple file paths, probably selected from the file_list form
 * @param $context A set of parameters, similar to the import_html profile, possibly from the list_filesystem form. Should contain the base path that the submitted files are relative to. 
 * 
 * @return A result set of nodes
 */
function import_html_import_files($file_list, $context) {
  drupal_set_message(t('Processing files!'));
#  dsm(count($file_list) ." files to go on");
#  dsm($file_list);
#  dsm(array('Context' => $context));

  if ( empty($file_list)) {
    drupal_set_message(t("No Files Selected. Nothing to import"), 'error');
    return;
  }

  // TODO see if we really need to keep a handle on all these result nodes at once.
  // It will be eating into our memory.
  $results = array();
  foreach ($file_list as $rel_path) {
    if ($rel_path) {
      $results = import_html_import_file($rel_path, $context);
      if (! $results) {
        drupal_set_message(t('Failed to get any results from the attempted analysis of %rel_path. The source file path was probably unavailable or incorrect.', array('%rel_path' => $rel_path)), 'error');
        return FALSE;
      }
      // result of importing a file MAY be more than one node, unlikely as it may be
      debug_pre(array( "Result of processing file $rel_path" => $results), 4);
      foreach($results as $node) {
        unset($node->file_data); // discard debug logs, try to save space.
        $results[] = $node;
      }
    }
  }
  // This isn't happening until I visit admin?
  menu_rebuild();
  return $results;
}

/**
 * Given a html file, prepare all the node info we can get out of it.
 * 
 * This func mainly prepares the paths and relative links. Data extraction happens in _import_html_process_html_page()
 * 
 * It does submit and save the node to the database.
 * 
 * @return an array that may contain more than one node (in extreme cases)
 */
function import_html_import_file($rel_path, $context) {
  
  // Read the profile id and use that as a context for all settings

  $profile = import_html_profile($context['profile_id']); 
  $base_path = $context['base_path'];
  $dest_root = $profile['file_storage_path'];
  $url_parts = parse_url($base_path);

  drupal_set_message(t("<strong>Importing</strong> '%rel_path'", array('%rel_path' => $rel_path)));

  $source_path = $base_path . $rel_path;
  $save_as = safe_filepath_from_url($rel_path);
  $dest_path = preg_replace("|/+|", "/", $dest_root . $save_as);

  if (preg_match("|/$|", $rel_path)) {
    // Handle trailing slashes differently at home and away
    if ($url_parts['host']) { 
      // It's remote
      $default_documents = split(",", $profile['default_document']);
      // need a dummy filename if retrieving default docs.
      $dest_path .= trim(array_shift($default_documents));; 
    }
    else {
      return; //skip local directories altogether (their contents is selected individually)
    }
  }
  
  $file = array(
    'source' => $source_path,
    'dest' => $dest_path,
    'rel_path' => $rel_path,
  );

  // Handle files that are resources.
  // Copy them into the files folder and return
  $checkfile = is_local($source_path) ? $source_path : $dest_path;
  // can't use mime detection on remote lookups yet
  if (import_html_guess_file_class($checkfile) != 'html') {
    // non-page resource - what sort of processing can I do here?
    drupal_set_message(t("
      I think (due to file suffix '%doctype') that '%source' is not a html page
      I can process.<br/> It's just been copied into '!dest'.",
      array(
        '%source' => basename($source_path),
        '!dest' => l($dest_path, $dest_path),
        '%doctype' => import_html_guess_file_class($checkfile),
      )
    ));
    import_html_get_raw_file_local($source_path, $dest_path, $url_parts['host']);
    $file['type'] = 'resource';
    $files[] = $file;
    return $files;
  }


  // Compare the alias path of this new page with what we've already got
  $new_path = _import_html_calc_path($rel_path);
  if (($normal_path = drupal_get_normal_path($new_path)) != $new_path) {
    // We recognise that alias, thus an item already exists in that path.
    if ($profile['handle_duplicates'] == IMPORT_HTML_SKIP) {
      drupal_set_message("We already have the URL alias '$new_path' in the system as '$normal_path'. According to import_html settings, this import is being skipped");
      return;
    }
    drupal_set_message("We already have '$new_path' in the system as '$normal_path'. Overwriting/updating it with the new import");
  }

  // Minor clean-up. Helps recover from crashes and prevents files getting renamed into file-01.etc
  if ( file_exists($dest_path) && ! $profile['keep_temp_files']) {
    unlink($dest_path);
  }

  if (! file_exists($dest_path) || ! $profile['keep_temp_files']) {
    if (!import_html_get_raw_file_local($source_path, $dest_path, $url_parts['host'])) {
      drupal_set_message(t("Failed to fetch a copy of %source_path into %dest_path", array('%dest_path' => $dest_path, '%source_path' => $source_path) ));
      return NULL;
    }
    debug(t("Fetched a %persistant local copy to %dest_path", array('%dest_path' => $dest_path, '%persistant' => ($profile['keep_temp_files'] ? 'persistant' : 'temporary') )), 1);
  }
  else {
    debug(t("Local copy exists at %dest_path", array('%dest_path' => $dest_path)), 1);
  }


  // We have a local copy now.
  // $node initialized and processed HERE. Produces a node OBJECT
  //
  if ($url_parts['host']) { 
    // Importing a remote file - as for demo
    // relinking will happen to point back at where it came from, not here
    $rel_path = $source_path; 
    debug("Relinking this source will point back to the original URL context!", 2);
  }

  $nodes = import_html_process_html_page($dest_path, $rel_path, $profile);
  
  // At this point, the node(s) are full of data, but not yet saved.

  // On rare occasions, (using xt:document) the processing can produce an ARRAY of nodes that need saving
  // Otherwise it's a node object

  if (!$nodes) {
    drupal_set_message(t("Failed to process file '%rel_path'", array('%rel_path' => $rel_path)), "error");
    return FALSE;
  }


  // We can immediately discard the source file - 
  // it should have been a temp copy made by import_html_get_raw_file_local() above
  if ( file_exists($dest_path) && ! $profile['keep_temp_files']) {
    unlink($dest_path);
  }
  
  if (! is_array($nodes)) { // cast into array for iteration anyway.
    $nodes = array($nodes);
  } 

  // Almost trivial loop (probably over 1 item)
  foreach ($nodes as $node) {
    // The node data object has been initialized
    // It may contain heaps of extra junk set in via a random absorbtion of elements in the XML import.
    // They will get ignored if not recognised.
    drupal_set_message(t("Processed page to extract content. Title: '{$node->title}' "));
    
    // figure if it's overwriting an existing path
    $node = import_html_merge_over_existing_node($node);

    // If processing in bulk, any error will stop any further processing.
    // (form_errors is non-null) need to clear it between each pass.
    unset($GLOBALS['form']);
    node_validate($node);

    if (!trim($node->body)) {
      form_set_error('body', t("No body content found in this node"));
    }

    $file['node'] = $node;

    // Finished prep, now save
    
    if ($context['form_id'] == 'import_html_demo_form') {
      // DO NOT actually save stuff to the database
      node_submit($node); 
    }
    else {

      if ($errors = form_get_errors()) {
        drupal_set_message(t("Import of '%rel_path' did not quite validate. I'm not sure how to recover from that problem. <br/>!errors", array(
          '%rel_path' => $file['rel_path'],
          '!errors' => join(',<br/> ', $errors),
        )), 'error');
        // TODO This is not very helpful in bulk mode.
        // what can I do now?
      }
      else {
        if ($node->nid) {
          drupal_set_message(t("!node_link Exists, updating it.", array('!node_link' => l('node '. $node->nid, 'node/'. $node->nid)) ));
          watchdog('Import HTML', t("Updating node !node_link with content from $source_path", array('%source_path' => $source_path, '!node_link' => l($node->path, $node->path))), WATCHDOG_NOTICE);
          node_save($node);
          module_invoke_all('import_html_after_save', $profile, $node);
        }
        else {
          drupal_set_message(t("Inserting New Node. ". l($node->path, $node->path)));
          watchdog('Import HTML', t("Inserting New Node !node_link with content from $source_path", array('%source_path' => $source_path, '!node_link' => l($node->path, $node->path))), WATCHDOG_NOTICE);
          $node = node_submit($node); 
          // Submit doesn't actually save, it just fills in extra fields
          node_save($node);
  
          // Had to wait until I had an ID to do this
          // These callbacks add the aliases and menus
          module_invoke_all('import_html_after_save', $profile, $node);
          // note, navigation only gets set up on first import. After that you are on your own
        }
      } // Finished updating database
      
      // and show the rendering of the new page teaser
      $file['teaser'] = node_view($node, TRUE);
    }
    $files[] = $file;
  }

#dpm(array('file will be' => $file));
#dpm(array('Ready to save processed node(s) is' => $files));

  return $files;    
}


/**
 * Files have been selected, set them up for processing
 * 
 * @param $file_list an array of simple file paths, probably selected from the file_list form
 * @param $context A set of parameters, similar to the import_html profile, possibly from the list_filesystem form. Should contain the base path that the submitted files are relative to. 
 */
function import_html_queue_files($file_list, $context) {
  if ( empty($file_list)) {
    drupal_set_message(t("No Files Selected. Nothing to import"), 'error');
    return;
  }
  foreach ($file_list as $rel_path) {
    if ($rel_path) {
      import_html_add_to_queue($rel_path, $context);
    }
  }
  return;
}

/**
 * Inserts the requested action into the daemon queue to be called later
 */
function import_html_add_to_queue($rel_path, $context) {
  $sql = "INSERT INTO {import_html_queue} (command, subject, context, date) VALUES ('%s', '%s', '%s', '%d')";
  db_query($sql, 'import_html_import_file', $rel_path, serialize($context), time());
  drupal_set_message(t("%rel_path added to processing queue for importing soon", array('%rel_path' => $rel_path)));
}


/**
 * Carefully fetch a (potentially remote?) file and save it nearby
 */
function import_html_get_raw_file_local($source_path, $dest_path, $host) {
  mkdirs(dirname($dest_path), FILE_CREATE_DIRECTORY);
  drupal_set_message(t("Fetching content from %location '<a href='!realpath'>%source_path</a>' now.", array(
    '%source_path' => $source_path,
    '%location' => $host,
    '!realpath' => realpath($source_path),
  )));
  debug("Saving temp file locally as '$dest_path' ", 3);

  $orig_path = $source_path;

  if ($host) { 
    // It's remote. Trust PHP5 and allor_url_fopen is available
    if (!copy($source_path, $dest_path)) {
      drupal_set_message(t("Remote file copy failed"));
      return FALSE;
    }
  }
  else { 
    // local copy
    if (realpath($source_path) == realpath($dest_path)) {
      drupal_set_message("Copying between identical source and destination, $source_path $dest_path , importing file in-place.");
      return TRUE;
    };
    // drupal file copy assumes everything is under 'files' directory
    #$dest_path = realpath(dirname($dest_path)) .'/'. basename($dest_path);

    if (!copy($source_path, $dest_path)) {
      drupal_set_message(t('Local file copy failed (%from to %to)', array('%from' => $orig_path, '%to' => $dest_path)), 'error');
      drupal_set_message("Source <code>$source_path</code> is <pre>". print_r(stat($source_path), 1) ."</pre>", 'error');
      drupal_set_message("Dest folder properties are <code>$dest_path</code> <pre>". print_r(stat(dirname($dest_path)), 1) ."</pre>", 'error');
      
      return FALSE;
    }
  }

  debug("Copied import file from '$orig_path' to '$dest_path'", 3);
  return TRUE;
}


/**
 * Analyse a source page and create a node definition from it.
 *
 * Most of the processing magic is in here.
 * The $node handle may be provided initialized with some pre-set values.
 * The $node may come in as an array or an object.
 * Internally we should continue using the object methods.
 *
 * This processing is still in the 'validate' phase, so should
 * not cause anything to happen, just configure the node object
 *
 * @param $path/$node the file (or object) to read the data from. If it's a
 * string, it's taken to be the filename, if an object, it's the node. A node
 * should contain a - >body (or ->raw_html) and a - >path at least.
 * @param $rel_path Where this html page was found, relative to its own server
 * root. This is used to rewrite its urls. If the path is a directory, it should
 * end with a slash. ( /a/path/ == /a/path/index.html != /a/path )
 * @return an ARRAY containing the new node object as the first item. Some
 * processes may return multiple nodes
 * @param $profile The settings for this import process.
 */
function import_html_process_html_page($path, $rel_path, $profile) {
  if (!init_xsl()) {
    trigger_error("Sorry, with no XML support there will be no content scanning AT ALL. Aborting process. See the import_html_help.htm for info on enabling XML under PHP.", E_USER_ERROR);
    return;
  }
  debug(t("import_html.module debugging is enabled to level %debug_level. Visit the <a href='!profile_config'>profile configuration</a> (advanced) to turn down the volume.", array('%debug_level' => $profile['debug_level'], '!profile_config' => url( IMPORT_HTML_ADMIN_PATH .'/profile/'. $profile['profile_id']))), 1);
  debug_pre(array($profile, "The import profile settings being used to import_html_process_html_page($rel_path)"), 2);

  if (is_string($path)) { 
    // read from file
    debug("Processing file as HTML page. Full file path: '$path' , will be imported as a relative path under the current section. relative-path is:'$rel_path'", 1);

    if (! file_exists($path)) {
      trigger_error("Path '$path' was not found. This should have been a local copy of the file being imported, but the paths may be wrong somehow. Abject failure processing $rel_path"); 
    }

    /*
     * Trying to parse pure XML first is causing problems
     * Either I want everything to be html, (always tidy)
     * or I allow for exsl:document blocks (which can't be tidied)
     * Option for now is try to parse, and only tidy if that fails.
     * 
     */
     // temporarily ignore parser errors (catch?)
    set_error_handler('stfu');
    $xmldoc = parse_in_xml_file($path, $profile['force_tidy']);
    restore_error_handler();
    
    if (! $xmldoc && $profile['force_tidy'] ) {
      debug("$path was not tidy enough - running tidy over it now so I can parse it.", 1);
      // If a raw XML parse failed,
      // tell parse_in_xml_file() to use htmlTidy before it begins
      // TODO - add a flag to skip this double-processing, (parsing twice) it may be a bit slow if it's not often used
      $xmldoc = parse_in_xml_file($path, TRUE);
    }
    debug_pre( array("Finished reading from file:" => xml_tostring($xmldoc)), 3);
    $source_node = new stdClass();
  }
  else { 
    // We may have passed in a source-node object where the path was expected instead.
    // A bit of a sneak. The given node has the source HTML in $node->raw_html
    if (is_object($path)) {
      $source_node = $path;
      $path = $source_node->path;
      if(! $source_node->raw_html) {
        trigger_error(t("import_html_process_html_page called with no HTML source to analyse"), E_USER_ERROR);
      }
      debug("Processing page source, ". strlen($source_node->raw_html) ." chars long", 2);
      debug_pre(array("Raw source" => $source_node->raw_html), 3);

      $xmldoc = parse_in_xml_string($source_node->raw_html, $profile['force_tidy']);
    }
  }
  if (!$xmldoc) { 
    // parsing failed
    drupal_set_message(t("Import_HTML failed to initialize or parse XMLdoc input"), "error");
    // Insert extra debug to see why
    $source_node->file_data['after_tidying'] = xml_tidy_string($source_node->raw_html);
    return false;
  }
  debug_pre(array("PARSED XML $path . XHTML" => xml_tostring($xmldoc)), 2);

  if ($profile['rewrite_links']) {
    // use XSL to rewrite links to fit into Drupal
    $xmldoc = import_html_rewrite_links($xmldoc, $rel_path, $profile);
  }
  if ($profile['strip_tables']) {
    $xmldoc = import_html_strip_tables($xmldoc);
  }
  if ($profile['strip_scripts']) {
    $xmldoc = import_html_strip_scripts($xmldoc);
  }
  if (TRUE || $profile['tag_editable_areas']) {
    $xmldoc = import_html_tag_editable_areas($xmldoc);
  }

  // Debug trace data
  if (import_html_variable('debug_level')) {
    $source_node->file_data['after_rewriting'] = xml_tostring($xmldoc);
  }
  
  // Import content as node.
  // Translate the source text to the known tidy simple, tagged HTML structure now
  $parameters = array(
    'xmlid' => TRUE,
  );
  if ( !empty($profile['content_tag_id'])) {
    $parameters['contentid'] = $profile['content_tag_id'];
  }

  if ($xsldoc = _import_html_get_xsl_doc($profile['translation_template'])) {
    debug("Using XSL translation template to extract semantic content. Will search for body content labelled '". $parameters['contentid'] ."' in the source. Active XML Namespaces are {$xmldoc->firstchild->nodename} : {$xmldoc->firstchild->namespaceuri} - {$xsldoc->firstchild->nodename} : {$xsldoc->firstchild->namespaceuri}  \n", 1);
    $importxml = xmldoc_plus_xsldoc($xmldoc, $xsldoc, $parameters);
    debug_pre(array("Transform Successful. TRANSLATED from messy source into a pure xhtml page to import" => $importxml), 2);
  }
  else {
    trigger_error("Failed to initialize XSLdoc", E_USER_WARNING);
  }

  if ($importxml) {
    $xmldoc = parse_in_xml_string($importxml, false);
    //
    // Allow one source document to produce multiple nodes
    // If the process has resulted in xt:document blocks, each block
    // is a new item.
    // Either there is a html element in the input ... or many of them.

    $html_elements = xml_getelementsbytagname($xmldoc, 'html');

    debug("Found ". count($html_elements) ." html elements in source doc", 3);

    $nodes = array();
    // probably only one, but we'll iterate over an array of one then
    foreach ($html_elements as $html_element) {
      $node = import_html_xhtml_to_node($html_element, $source_node, $profile);

      // Set what we want the alias to be.
      if (! $node->path) {
        $node->path = _import_html_calc_path($rel_path);
        $node->old_path = _import_html_calc_path($rel_path, TRUE);
      }

      // May need extra care when creating multiples. 
      // Invent new paths for the new documents if the exsl:document didn't define them
      if ($nodes[$node->path]) {
        // already using this path, extend a new one  
        $node->path .= '/'. import_html_check_name($node->label?$node->label:$node->title);
      }

      $node->title = import_html_guess_document_title($node);

      $node->status = $profile['import_status'];
      $node->promote = $profile['import_promote'];

      // Tag this new content if the profile has a global tag set  
      $cats = $profile['import_category'];
      if (is_array($cats)) {
        foreach ($cats as $cat) {
          $node->taxonomy[$cat] = taxonomy_get_term($cat);
        }
      }
      
      // debug notes/trace logs. Can be removed
      if (import_html_variable('debug_level')) {
        $node->file_data['raw_xhtml'] = xml_toString($html_element);
      }
      # $node->xml = $html_element;

      $nodes[$node->path] = $node;
      
      debug("Path to save this page as is '". $node->path ."'", 1);
    }    
  }
  else {
    trigger_error("Nothing useful extracted via XML from that content", E_USER_WARNING);
    return false;
  }
  debug_pre(array("PROCESSED node body" => $node->body), 3);
  return $nodes;
}


/**
 * From a given XML document, create a node structure
 * with all useful parameters set.
 * A shell node object may be passed in with some values already set. The data
 * extracted from the XHTML structure will be layered onto that.
 * 
 * Here is where we map HTML info to node data, like H1 -> $node->title
 * TODO tidy this up with a lookup table or something
 * 
 * node may have defined its own $node->type even
 * 
 * THIS IS THE ENGINE OF IMPORT_HTML
 */
function import_html_xhtml_to_node($datadoc, $node, $profile) {
  debug("Importing from XML object to node object", 3);
  $node = $node ? $node : new stdClass();
  $node->type = is_string($node->type) ? $node->type : $profile['content_type'];
  
  $node->taxonomy = is_array($node->taxonomy) ? $node->taxonomy : array();
  
  // Now read the input into node structure
  //
  // Absorb the most generic bits first. Later processes may overwrite them more accurately.
  
  // This initial import is a totally generic catch-all. 
  import_html_absorb_all_tagged_elements(&$node, $datadoc);

  //
  // Get all metas as properties
  //
  $head_element = xml_getelementsbytagname($datadoc, 'head', TRUE);
  // Allow ALL values I find (some may get lost later)
  import_html_absorb_metas($node, $head_element, 'meta', 'name', 'content');
  import_html_absorb_metas($node, $head_element, 'link', 'rel', 'href');

  // If there are any other things to come from HTML into $node, let me know now!
  // Loop over a buch of hook-like per-module extensions
  // MENU, PATH, TAXONOMY, CCK all add values in their own callbacks in import_html_modules.inc
  // Also the core node elements - body, title, teaser get set in a callback
  //
  require_once('import_html_modules.inc');
  import_html_include_add_on_module_handlers();
  
  module_invoke_all('import_html', $profile, $node, $datadoc); 
   
  // 'content' is now a reserved word in Drupal5
  // If I have a string there, the body cannot be rendered right later
  unset($node->content);

  // The preferred filter 'format' of this body is none - not even line breaks
  $node->format = import_html_get_preferred_filter();

  debug("After absorbing absolutely everything I could find, the node object now contains the following blocks and bits:", 3);
  debug_pre(array( 'Absorbed all node structure from the XHTML. Node is:' => $node), 2);
  return $node;
}

/**
 * Import ALL tagged classes and IDs as node attributes.
 * 
 * If the input has ANY id or classes at all, grab that info and apply it to
 * this object. Assume anything important enough to have a label is important
 * enough to remember.
 * 
 * This will probably produce a very cloggy node, filled with trash, Possibly
 * even some arrays where there shouldn't be. But any unrecognised property
 * names will be discarded on save, leaving only the serializable values. This
 * approach will allow arbitrary data to come and go in the future.
 *
 */
function import_html_absorb_all_tagged_elements(&$node, $datadoc) {

  foreach (array('id', 'class') as $attribute_label) {
    
    debug("Absorbing all blocks with an $attribute_label as incidental data blobs (possibly html) into node structure", 3);
    $found_elements = xml_query($datadoc, './/*[@'. $attribute_label .']');

    // I now have a collection of tagged nodes.    
    foreach ($found_elements as $found_element) {
 
      $attribute_value = xml_getattribute($found_element, $attribute_label);
      // if it was a class, it may be multiple! 
      // Usually just one however...
      $keys = explode(' ', $attribute_value); 
      // debug("Found an node with $attribute_label of ".print_r($keys, 1) , 3);
 
      foreach ($keys as $key) {
        // Found 'something' labelled 'something'
        if(! trim($key)) {continue;}
        // Allow HTML though. Sometimes this will not be right...
        // TODO, figure it out?
        $value = xml_tostring($found_element, TRUE);
        if(! trim($value)) {continue;}

        // The value just gets absorbed
        debug("Found an unexpected tagged value - '$key' , Absorbing it into the node as a default text/html value", 2);

        // Set it onto the node, 
        // If it's a class, carefully combine to preserve pre-existing arrays
        if ( $attribute_label == 'class') {
          import_html_absorb_properties($node, $key, $value);
        }
        else {
          // but if it's an ID, there can be only one, just set it
          $node->$key = $value;
        }
      } // each multiple key
    } // each found element
  } // each attribute type
}  

function import_html_absorb_metas(&$node, $xml_element, $tagname, $keyname, $valname) {
  $metas = xml_getelementsbytagname($xml_element, $tagname);
  debug("Absorbing the '$valname' of '{$tagname}'s with a '$keyname' from source doc into node structure", 3);
  foreach ($metas as $meta) {
    if (empty($meta)) {continue;}
    $key = xml_getattribute($meta, $keyname);
    $value = xml_getattribute($meta, $valname);
    if ($key && $value) {
      import_html_absorb_properties($node, $key, $value);
      if (module_exists('nodewords')) {
        $node->nodewords[strtolower($key)] = $value;
      }
    }
    else{
      debug("When absorbing '$valname' from '{$tagname}'s with a '$keyname' from source doc ($key=$value) had a null value. Not a great problem, just letting you know.", 2);
    }
  }
}

/**
 * Include what we can find in the /modules directory.
 * Only once.
 */
function import_html_include_add_on_module_handlers() {
  static $done;
  if ($done) return;
  // Scan add-on dir and include all bits found there
  $inc_files = file_scan_directory(drupal_get_path('module', 'import_html') .'/modules', ".*.inc", array('.', '..', 'CVS')) ;
  foreach($inc_files as $inc_path) {
    include_once($inc_path->filename);
  }
  $done = TRUE;
}

/**
 * Set the given property on the given object,
 * allowing multiple values to expand into arrays.
 *
 * Happens automatically IFF more than one kay match is found. Deal with that
 * yourself.
 */
function import_html_absorb_properties(&$node, $key, $value) {
  if (!$key) {debug("Odd, when absorbing properties, value:'$value' is a value for what key? The calling function passed a null key to be absorbed.");return;}
  if (!$value) {debug("Odd, when absorbing properties, '$key' had a null value. This is probably not an error.", 2);return;}
  
  // auto-expand into arrays - most metas can legally have duplicates
  if ( ! isset($node->$key) ) {
    $node->$key = $value;
  }
  else if ( is_array($node->$key) ) {
    $a = $node->$key; $a[] = $value; $node->$key = $a; 
  }
  else { $node->$key = array($node->$key, $value); }
}


function import_html_guess_document_title($node) {
    if (! $node->title ) {
      switch (import_html_variable('handle_no_title')) {
        case IMPORT_HTML_GUESS :
          return import_html_guess_label($node->path);
          break;
        case IMPORT_HTML_DEFAULT :
          return 'Untitled Document';
          break;
      }
    }
  return $node->title;
}

function import_html_guess_label($title) {
  $path_bits = split('/', $title);
  $title=array_pop($path_bits);
  if (!$title) {$title=array_pop($path_bits);} // it had a trailing slash
  $title = str_replace('_', ' ', $title);
  $title = (strstr($title, '.')) ? substr($title, 0, strrpos($title, ".")) : $title;
  return $title;  
}


/**
 * Return the nice path alias of an imported page
 */
function _import_html_calc_path($rel_path, $leave_suffix = FALSE) {
  $path = import_html_variable('import_site_prefix') . preg_replace('|^/|', '', $rel_path);
  $path = preg_replace('| |', '%20', $path); // URLs should NOT have spaces, but old sites may have done this

  if ($leave_suffix) {
    return $path;
  }

  if (import_html_variable('trim_suffixes')) {
    // Simplify the URL if possible by trimming the suffix and 'index'
    // but remember the original path somewhere, we'ill need to link it forward
    // once the new node is established. 

    // To be clever, special-case the 'index.html' files to be
    // linked to their parent directories.
    // Trailing slash is tricky.
    // /this/path is a whole navigation level above
    // /this/path/ and will resolve relative links differently! 
    // We need to actually redirect, not just alias any links like that
    $default_documents = split(",", import_html_variable('default_document'));
    $trimmed_path = $path;
    foreach($default_documents as $default_document) {
      $trimmed_path = preg_replace('|/('. trim($default_document) .')$|', "", $trimmed_path);
    }
    if ($trimmed_path != $path) {
      debug("It's an index page, so we will refer to $path as $trimmed_path", 2);
      $path = $trimmed_path;
    } 
    else {
      // No change, Chop suffix instead
      $path = preg_replace('|\.[^\.]+$|', "", $path);
    }
  }
  return $path;
}

/**
 * Find and initialize the transformation template. Caching retrieval
 */
function _import_html_get_xsl_doc($xslfile) {
  static $xsldoc;
  if ($xsldoc) {
    return $xsldoc;
  }

  // Check if and where filepath can be found
  // Search first under full path, then module dir, then under files dir
  $xslfilepath = $xslfile;
  if (!file_exists($xslfilepath)) {
    #dpm("Did not find $xslfilepath, trying module dir");
    $xslfilepath = drupal_get_path('module', 'import_html') ."/$xslfile";
  }
  if (!file_exists($xslfilepath)) {
    $xslfilepath = file_directory_path() ."/$xslfile";
  }

  if (file_exists($xslfilepath)) {
    debug("Loading Transformation Stylesheet from $xslfilepath", 2);
    $xsldoc = parse_in_xml_file($xslfilepath, false);
  }
  else {
    drupal_set_message("Unable to locate the Transformation Stylesheet '$xslfilepath' ", "error");
    return false;
  }
  return $xsldoc;
}

/**
 * Run the url-rewrite XSL over the source document
 * TODO allow for the non-base version of Drupal links
 * 
 * The relative links need to be converted into path-to- top and back down
 * again. Relative references just cannot be maintained.
 * 
 * @return an XML doc again
 */
function import_html_rewrite_links($xmldoc, $rel_path, $profile) {
  static $rewrite_xsldoc; // memo this to speed up bulk imports
  static $xslfilepath;
  if (!$rewrite_xsldoc) {
    $xslfilepath = drupal_get_path('module', 'import_html') ."/rewrite_href_and_src.xsl";
    $rewrite_xsldoc = parse_in_xml_file($xslfilepath, false);
  }

  debug("Rewriting links for a file called '$rel_path'. dirname($rel_path) is ". dirname($rel_path), 2);
  debug_pre(array("import_html profile settings used for rewriting" => $profile), 3);

  // dirname('/ok.htm') returns '\'; No idea why, may only happen at root level on Win
  // !! B-X

  // $rel_base is the path from the import root to the current page dir
  // I want a trailing slash, but not a leading one for the next concatenation
  // dirname('/a/dir/') returns '/a' - which is not what I want
  $rel_dir = preg_match('|/$|', $rel_path) ? $rel_path : dirname($rel_path);
  $rel_base = ensure_trailing_slash($rel_dir);

  $site_root = url('');
  $path_to_import_top = url( ensure_trailing_slash($profile['import_site_prefix']) );
  $site_root = $path_to_import_top;

  // if we are re-writing thing/index.htm to thing - our links will resolve differently!
  // either too high for thing, or too low for the thing/index.htm alias.
  $href_base = url( ensure_trailing_slash($profile['import_site_prefix']) . $rel_base);

  // Create the prefix for resource sources
  // Is url() OK for files with unclean urls? - NO. Neither is file_create_url
  $src_root = base_path() . $profile['file_storage_path'];

  $src_base = ensure_trailing_slash($src_root) . ($rel_base == '/') ? '' : $rel_base;
  
  // Or not, if we are still linking to full URLs (demo or partial import)
  $url_parts = parse_url($rel_path);
  if ($url_parts['host']) {
    // it's remote!
    $path_to_import_top = $rel_path;
    $site_root = 'http://'. $url_parts['host'] .'/';
    $src_root = $site_root;
    $src_base = $rel_path;
  }
  $src_base = str_replace('/./', '/', $src_base);
  $href_base = str_replace('/./', '/', $href_base);


  debug("
    <b>Rewrite patterns:</b>
    Path to the top of this (relative) server is $site_root .
    Path to top of the prefixed section 
    ({$profile['import_site_prefix']})
    from here ($rel_path)
    to our import base 
    ({$profile['import_site_prefix']})
    would be '$path_to_import_top'.
    Path to a relative <em>neighbour</em> of this page would be
    ($href_base)
    or to find the base for <em>relative</em> resource files over in
    the file storage area
    ({$profile['file_storage_path']})
    would be '$src_base' ", 2
  );
  
  
  $parameters = array(
    // These parameters tell the rewriter what to prepend to the links.
    // They are instructions how this page will find its missing bretheren
    // when we put it where we put it.
    // Images and Pages may end up in different places.
    'site_root'      => $site_root,
    'src_root'       => $src_root, 
    'src_base'       => $src_base, 
    'href_base'      => $href_base,
    'replace_suffix' => $profile['relink_files'], 
    'new_suffix'     => '',
    'xsl_path'       => $xslfilepath,
    'strip_script_tags' => $profile['strip_script_tags'], 
  );
  debug("
    XSL for URL rewrites loaded OK. 
    HTML links for files that were under '$rel_base' will be made relative to '" 
    . $parameters['href_base'] ."' and '". $parameters['src_base'] ."'" 
    . ( $parameters['strip_script_tags'] ? 'All inline script blocks will be discarded from the source.'. $parameters['strip_script_tags'] : '')
    , 2);
  debug_pre(array("PARSED XSL $xslfilepath . XSL" => xml_tostring($rewrite_xsldoc)), 4);

  $rewritten = xmldoc_plus_xsldoc($xmldoc, $rewrite_xsldoc, $parameters);

  // collapse dir-up "../" paths. To tricky for XSL. Hope it doesn't break anything
  $rewritten = preg_replace('|/[^\.][^/\s"\'>]*/\.\./|', '/', $rewritten);

  debug_pre(array("The source after URL rewriting . XHTML (string)", $rewritten), 2);

  $xmldoc = parse_in_xml_string($rewritten, FALSE);
  if (empty($xmldoc)) {
    trigger_error("Failed to rewrite links into a valid XML file", E_USER_WARNING);
    return FALSE;
  }

  debug_pre(array(xml_tostring($xmldoc) => "Parsed in again. XHTML (XML)"), 3);
  return $xmldoc;
}

/**
 * Run the strip_tables XSL over the source document
 * 
 * @return an XML doc again
 */
function import_html_strip_tables($xmldoc) {
  static $strip_tables_xsldoc; // memo this to speed up bulk imports
  if (!$strip_tables_xsldoc) {
    $xslfilepath = drupal_get_path('module', 'import_html') ."/strip_tables.xsl";
    $strip_tables_xsldoc = parse_in_xml_file($xslfilepath, FALSE);
  }

  debug_pre(array("PARSED strip_tables XSL $xslfilepath . XSL" => xml_tostring($strip_tables_xsldoc)) , 3);
  $parameters = array();
  $rewritten = xmldoc_plus_xsldoc($xmldoc, $strip_tables_xsldoc, $parameters);

  // normalize space to clean up the gaps
  $rewritten = preg_replace("/\\s*\\n\\s*/", "\n", $rewritten);

  debug_pre(array("The source after stripping tables . XHTML (string)" => $rewritten), 3);
  $xmldoc = parse_in_xml_string($rewritten, false);
  if (!$xmldoc) {
    trigger_error("Failed to strip tables and end up with a valid XML file", E_USER_WARNING);
    return false;
  }

  return $xmldoc;
}

/**
 * @see import_html_strip_tables
 */
function import_html_strip_scripts($xmldoc) {
  static $strip_scripts_xsldoc; // memo this to speed up bulk imports
  if (!$strip_scripts_xsldoc) {
    $xslfilepath = drupal_get_path('module', 'import_html') ."/strip_scripts.xsl";
    $strip_scripts_xsldoc = parse_in_xml_file($xslfilepath, false);
  }
  $parameters = array();
  $rewritten = xmldoc_plus_xsldoc($xmldoc, $strip_scripts_xsldoc, $parameters);

  $xmldoc = parse_in_xml_string($rewritten, false);
  if (!$xmldoc) {
    trigger_error("Failed to strip tables and end up with a valid XML file", E_USER_WARNING);
    return false;
  }
  return $xmldoc;
}

/**
 * Use XSL to convert Dreamweaver 'instanceEditable' comments into semantic
 * tagged divs
 * 
 * @see import_html_strip_tables
 */
function import_html_tag_editable_areas($xmldoc) {
  static $editable_xsldoc; // memo this to speed up bulk imports
  if (!$editable_xsldoc) {
    $xslfilepath = drupal_get_path('module', 'import_html') ."/tag_editable_areas.xsl";
    $editable_xsldoc = parse_in_xml_file($xslfilepath, false);
  }
  if (!$editable_xsldoc) {
    trigger_error('Cannot tag_editable_areas. tag_editable_areas.xsl unavailable', E_USER_WARNING);
    return NULL;
  }
  $parameters = array();
  $rewritten = xmldoc_plus_xsldoc($xmldoc, $editable_xsldoc, $parameters);

  $xmldoc = parse_in_xml_string($rewritten, false);
  if (!$xmldoc) {
    trigger_error("Failed to tag commented editable areas (eg from Dreamweaver)  and end up with a valid XML file", E_USER_WARNING);
    return false;
  }
  return $xmldoc;
}


/**
 * Ensure a sting is able to be used as an XML, CSS or Javascript ID.
 * Basically strip out all non-alpha-numerics
 * http://www.w3.org/TR/REC-xml/#NT-Name
 * @see form_clean_id() - which should have done this
 */
function import_html_check_name($name) {
  return preg_replace('|[^a-zA-Z0-9_]+|', '_', $name);
}


/**
 * Avoid double-ups, if the path already exists, UPDATE the existing node.
 * Can't have two content nodes claiming the same path or it won't validate.
 * Plus, we want to retain any info that's been added via drupal. Probably.
 * 
 * @param $node - partially created node from import. Key lookup on $node->path
 * @return $node - possibly with pre-existing values blended in. Importantly - the nid
 */
function import_html_merge_over_existing_node($node) {
  $internal_link = drupal_get_normal_path($node->path);

  if ($internal_link != $node->path) {
    // Found an internal match, the alias is already asigned to a node
    // Merge info to avoid losing any Drupal-only info
  
    $node->nid = array_pop(explode("/", $internal_link));
    if (!$node->nid) {
      // Should never happen - just paranoia
      drupal_set_message("
        When looking for an alias to '{$node->path}', 
        got error finding node ID from the internal link 
        '$internal_link' - which was supposed to return a nid", 'error'
      );
    }
    else {
      debug("
        Page path alias '{$node->path}' already exists, 
        It's already linked to node id '{$node->nid}'. 
        This data import will <em>replace</em> that content, 
        but try to keep any other values.
      ", 2);
  
      // Load existing item, layer changes on top of it
      $old_node = node_load($node->nid );
      foreach ($node as $key => $value) {
        if (is_array($value)) { // merge deeper sets, like taxonomy
          if (!is_array($old_node->$key)) {$old_node->$key=array();}
          foreach ($value as $k => $v) {
            $old_node->{$key}[$k] = $v;
          }
        } 
        else {
          $old_node-> $key = $value;
        }
      }
      $node = $old_node;
    }
  }
  return $node;
}


/**
 * Tidy URLs before saving locally - for URL imports
 * 
 * Squash/hash query strings, but don't discard them.
 * Do discard fragment ids
 * 
 * Replace spaces and non-alphanumerics with underscore
 */
function safe_filepath_from_url($rel_path) {
  $save_as = preg_replace("|\?|", "%3f", $rel_path);
  $save_as = preg_replace("|\&|", "%26", $save_as);
  $save_as = preg_replace("|#.*|", "", $save_as);

  if (import_html_variable('allow_bad_urls')) {
    return $save_as;
  }

  $save_as = preg_replace("|[^A-Za-z0-9_\-~\./%]+|", "_", $save_as);
  return $save_as;
}

/**
 * http://nz2.php.net/manual/en/function.utf8-decode.php#85034
 */
function charset_decode_utf_8 ($string) {
  /* Only do the slow convert if there are 8-bit characters */
  /* avoid using 0xA0 (\240) in ereg ranges. RH73 does not like that */
  if (! ereg("[\200-\237]", $string) and ! ereg("[\241-\377]", $string)) {
    return $string;
  }

  // decode three byte unicode characters
  $string = preg_replace(
    "/([\340-\357])([\200-\277])([\200-\277])/e",       
    "'&#'.((ord('\\1')-224)*4096 + (ord('\\2')-128)*64 + (ord('\\3')-128)).';'",   
    $string
  );

  // decode two byte unicode characters
  $string = preg_replace(
    "/([\300-\337])([\200-\277])/e",
    "'&#'.((ord('\\1')-192)*64+(ord('\\2')-128)).';'",
    $string
  );

  // dman
  // Encode medium-high entities (>#128;). htmltidy is failing to catch (copyright) 0xA9 (#159)  symbol. Others?
  $string = preg_replace(
    "/([\200-\377])/e",
    "'&#'.ord('\\1').';'",
    $string
  );

  return $string;
}


/**
 * dummy error handler
 * Used to shush DOM errors when we know the doc is probably invalid
 */
function stfu($err, $str) {
  # debug($str, 4);
};