modules/import_html/import_html_ui.inc

<?php
/**
 * @file
 * Forms and UI functions for import_html.
 * 
 * In preparation for D6, UI, call-once funcs are in this include file,
 * although for D5, there is no win.
 * 
 * @ingroup import_html Import HTML
 * @author Dan Morrison http://coders.co.nz/
 * @version $Id: import_html_ui.inc,v 1.11.2.2 2009/03/14 03:18:51 dman Exp $ 
 */
 
///////////////////////////////////////////
// SETTINGS and PROFILES 

/**
 * Display the import_html options and settings.
 *
 * Settings menu callback
 * 
 * Display the default profile
 */
function import_html_admin_settings() {
  $form = array();
  // check htmltidy is present and correct
  include_once 'tidy-functions.inc';
  $form['HTMLTidy'] = html_tidy_settings();  

  // Warn about known problem with open_basedir
  if ($open_basedir = ini_get('open_basedir')) {
    $form['open_basedir']=array(
      '#value' => t("<p><strong>Warning:</strong> This server has open_basedir restrictions set [%open_basedir]. It's possible that external source directories may be inaccessible. <a href='%help_url'>Check the help for more details</a>.</p>", array('%open_basedir' => $open_basedir, '%help_url' => url('admin/help/import_html', '', 'open_basedir')))
    );
  }

  $profile_id = import_html_current_profile_id();
  $form['default_profile'] = import_html_profile_settings($profile_id);
  $form['#submit'] = array('import_html_profile_settings_submit' => array()); 
  $form['#validate'] = array('import_html_profile_settings_validate' => array()); 

  $form['import_html_proceed'] = array(
   '#value' => '<h2>'. l(t("Proceed to import"), IMPORT_HTML_ADMIN_PATH .'/import_site') .'</h2><br/>',
  );

  // Links to other profiles
  $form['other_profiles'] = array(
    '#type' => 'markup',
    '#value' => '<h3>'. t('Other Profiles:') .'</h3>'. import_html_profiles_page(),
  );

  return $form;
}


/**
 * Page to display a list of current profiles, or to edit a named one
 */
function import_html_profiles_page($profile_id = null) {
  if (!$profile_id) {
    // Just return links to all other profiles
    $import_html_profiles = variable_get('import_html_profiles', array());
    $links = array();
    foreach ($import_html_profiles as $profile_id => $profile) {
      $links[] = array(
        'title' => $profile_id,
        'href' => IMPORT_HTML_ADMIN_PATH .'/profile/'. $profile_id,
      );
    }
    $links[] = array(
      'title' => t('New'),
      'href' => IMPORT_HTML_ADMIN_PATH .'/profile/'.'new',
    );
    return theme('links', $links);
  }
  // Else, render this given profile
  return drupal_get_form('import_html_profile_settings', $profile_id);
}


/**
 * An import_html profile settings form.
 *
 */
function import_html_profile_settings($profile_id) {

  $profile = import_html_profile($profile_id);
  if (! $profile) {
    // Fill in defaults (should only be needed first time, if that
    $profile = import_html_profile_defaults(); 
  }

  $form = array(); 

  if ($profile_id != 'new') {
    $form['summary'] = array('#value' => import_html_profile_summary($profile));
  }
  $form['profile_id'] = array(
    '#title' => t("Profile ID"),
    '#type' => 'textfield',
    '#required' => true,
    '#default_value' => '',
  );
  if ($profile_id != 'new') {
    $form['profile_id']['#default_value'] = $profile_id;
    $form['profile_id']['#disabled'] = true;
  }

  $form['extraction'] = array(  
    '#type' => 'fieldset',
    '#title' => t('Import and Content Analysis Options'),
    '#collapsible' => TRUE,
    '#collapsed' => TRUE,
  );
  $form['extraction'][] = array( 
    '#value' => t("How the content is extracted from the source documents."),
  );

  // This is abstracted so it can be re-used in other places (wrapper.module)
  $form['extraction']['translation_template'] = import_html_template_selector($profile['translation_template']);

  $form['extraction']['content_tag_id'] = array(
    '#type' => 'textfield',
    '#title' => t('Content Tag ID'),
    '#default_value' => $profile['content_tag_id'],
    '#size' => 20,
    '#maxlength' => 100,
    '#description' => t("
      If your source document is structured well enough to have the actual
      content in a named div or something (&lt;div id='main'&gt;...content )
      what is that name?
      <br/>
      The default template will search for 'main', 'copy', 'story', 'text', and
      'bodyCopy'. If your input uses another label, set it here.
      <br/>
      This only takes effect if the selected XSL template uses this parameter,
      and may be overridden or ignored by custom templates.
    "),
  );


  $form['extraction']['content_type'] = array(
    '#type' => 'select',
    '#title' => t("Node Type for new pages"),
    '#options' => node_get_types('names'),
    '#default_value' => $profile['content_type'],
    '#description' => t("
      Note, if importing into a CCK content type, or anything that doesn't 
      store its body in the node 'body', some customization will have to be
      done on your XSL template.
    ")
  );

  // Just a select, the filter radios are much to bulky for here.
  $form['extraction']['preferred_filter'] = array(
    '#type' => 'select',
    '#title' => t("Default Input Filter"),
    '#default_value' => $profile['preferred_filter'],
    '#options' => import_html_get_preferred_filter(TRUE),
    '#description' => t("
      After importing HTML, the end result may still have a standard Drupal 
      filter applied to it internally. 
      Use 'Unfiltered HTML' if you want the content verbatim.
      'Full HTML' <em>may</em> be correct here, although it may create 
      unwanted hard breaks if the input is already word-wrapped internally. 
      Modify only if you want to strip the source right down, execute php,
      or do other filter magic on the input.
    "),
  );

  $form['extraction']['default_document'] = array(
    '#type' => 'textfield',
    '#title' => t("Default Document"),
    '#default_value' => $profile['default_document'],
    '#size' => 40,
    '#maxlength' => 40,
    '#description' => t("
      What is the default document name for pages served in directories on
      the current site (index.htm, index.html, default.asp, index.php)?
      When documents of this name are imported, they can be given both
      the old name (my/dir/index.htm) and a friendly directory name (my/dir) .
      When the navigation menu is built, this file is served when the
      directory link is clicked.
      This should end up emulating normal website navigation behaviour.
      <br/>
      May be a comma-separated list, the first option is the default.
    "),
  );

  
  $form['extraction']['file_exclusions'] = array(
    '#type' => 'textarea',
    '#title' => t("File Exclusion Pattern"),
    '#default_value' => $profile['file_exclusions'],
    '#description' => t("
      When listing files to import, some sources should probably be left behind.
      Enter a set of regular expressions to check against the file <b>path</b>
      that should <em>not</em> show up on the import screen.
      <ul><li>
      <code>^_</code> = begins with '_'
      </li><li>
      <code>/_</code> = in a dir that begins with '_'
      </li><li>
      <code>CVS/?</code> = called CVS, or has CVS in the path
      </li></ul>
    "),
  );

  $form['extraction']['strip_tables'] = array(
    '#type' => 'checkbox',
    '#title' => t("Remove table markup"),
    '#return_value' => TRUE,
    '#default_value' => $profile['strip_tables'],
    '#description' => t("
      A common task when updating old sites is removing the
      legacy formatting tables. While we are going through the
      pages, this process can do it all for you.
      Beware - enabling this will remove ALL tables, so don't
      try it if there are genuine data tables you wish to retain!
    "),
  );
  $form['extraction']['strip_scripts'] = array(
    '#type' => 'checkbox',
    '#title' => t("Remove old javascripts"),
    '#return_value' => TRUE,
    '#default_value' => $profile['strip_scripts'],
    '#description' => t("
      There are many reasons why importing old inline javascripts
      may be a bad idea. Doing so will remove some functionality
      but also clean up some crap. Up to you.
    "),
  );
  $form['extraction']['force_tidy'] = array(
    '#type' => 'checkbox',
    '#title' => t("Force HTML Tidy-up"),
    '#return_value' => TRUE,
    '#default_value' => $profile['force_tidy'],
    '#description' => t("
      Run HTML Tidy with all strict options over the input before 
      parsing it as XML. Normally you want this on BUT turn it off
      if trying to read from non-HTML sources (like arbitrary XML)
      or it will mutilate it.
      You can feed in any valid XML and use XSL template to massage
      it into simple XHTML, but in that case do NOT use HTML tidy.
    "),
  );

  
  $form['replication'] = array(
    '#type' => 'fieldset',
    '#title' => t("Replication Options"),
    '#collapsible' => TRUE,
    '#collapsed' => TRUE,
  );
  $form['replication'][] = array( 
    '#value' => t("How and where the imported content is reproduced on this site.")
  );

  if (!module_exists('path')) {
    drupal_set_message(t("The <b>path</b> module is not enabled! If you try to import pages without paths, I can do it, but links won't work and you'll find it hard to find the pages again.'"), 'error');
  }

  $form['replication']['trim_suffixes'] = array(
    '#type' => 'checkbox',
    '#title' => t("Trim Suffixes"),
    '#return_value' => TRUE,
    '#default_value' => $profile['trim_suffixes'],
    '#description' => t("
      When importing files, new url_aliases will be generated based on the
      old path. Optionally, the new primary alias can have the suffix
      chopped of. So <code>old/path/file.htm</code> wil be accessed as just
      <code>old/path/file</code> and <code>old/section/index.htm</code>
      will become <code>old/section</code>.
      Uncheck this if you do NOT want this tidy behaviour.
      <br/>Note, this aliasing is needed to detect when an import is
      <em>replacing</em> a previously imported page. Disabling it may
      cause you to end up with duplicates if importing the same file again.
      <br/>This just defines the name given to the page, it does not
      automatically update any links <em>to</em> the page (see below) .
    "),
  );

  $form['replication']['legacy_aliases'] = array(
    '#type' => 'checkbox',
    '#title' => t("Legacy Aliases"),
    '#return_value' => TRUE,
    '#default_value' => $profile['legacy_aliases'],
    '#description' => t("
      Optionally, a secondary alias can be made that DOES have the old
      full suffix. This supports old links and hrefs.
      <b>The new node may end up with two aliases</b> (thats OK) but the tidy
      one will be used internally by default.
      Uncheck this if you do NOT want old URL support.
      <br/>
      These two options are independant of each other. If both are off
      the new page will have no url_alias.
    "),
  );
  
  $form['replication']['relink_files'] = array(
    '#type' => 'checkbox',
    '#title' => t("Try to Relink Files by discarding suffixes"),
    '#return_value' => TRUE,
    '#default_value' => $profile['relink_files'],
    '#description' => t("
      <b>If</b> you choose not to support the Legacy Aliases suffixes
      on your new pages, <em>existing embedded links to those pages will break</em>.
      This includes inline links from within the content we are importing.
      <br/>If a page linked to /home/about.htm and you have renamed it
      to home/about, without legacy support, the old link will be broken.
      This tool is a batch job process, not an entire site spider and
      relinker.
      <br/>
      <b>But</b> if you understand exactly what is going on,
      you can choose to try and rewrite all local links to their shortname
      equivalent (basically dropping off the suffixes of linked files)
      during import.
      This will convert any existing <code>href=\"local/path/file.htm\"</code>
      into just <code>href=\"local/path/file\"</code>. Use with care, but
      this option plus \"Trim Suffixes\" should be able to work together to
      rename all files. You\'ll want to run a link checker afterwards!
    "),
  );

  $form['replication']['allow_bad_urls'] = array(
    '#type' => 'checkbox',
    '#title' => t("Allow bad URLs and filenames"),
    '#return_value' => TRUE,
    '#default_value' => $profile['allow_bad_urls'],
    '#description' => t("
      It's bad mojo to have spaces or non-alphanumeric characters
      in your filenames and in your URLs. Normally I try to fix this
      by replacing potentially damaging characters with an underscore.
      If, however you have to support legacy stuff like 
      'Random filename #3 - & about what?.pdf' then we can try to leave them alone.
      May not always work the way you expect. 
    "),
  );

    
  $form['replication']['file_storage_path'] = array(
    '#type' => 'textfield',
    '#title' => t("Extra File Storage Path"),
    '#default_value' => ensure_trailing_slash($profile['file_storage_path']),
    '#description' => t("
      Where, beneath siteroot, should extra imported <strong>files</strong>
      be stored. Images and suchlike will be put into a heirachy mirroring
      their old location, but all under this directory.
      <br/>
      Effectively, this moves file storage towards the Drupal CMS way of 
      storing user files, although it does not enter them as 'attachments'.
    "),
  );
  $form['replication']['rewrite_links'] = array(
    '#type' => 'checkbox',
    '#title' => t("Relink references to resources to the new file storage location"),
    '#return_value' => TRUE,
    '#default_value' => $profile['rewrite_links'],
    '#description' => t("
      As the actual files are being stored elsewhere, the HTML links need to be
      rewritten to allow for that. This should be on by default. 
      Turn off only if you are intending to re-create a 
      messy site with files in old places and have set the file storage path to blank. 
      See the transformation file 
      <code>rewrite_href_and_src.xsl</code> for fine-tuning if there is trouble.
      <small>Only relative and site-relative links are rewritten. 
      Hard-coded, fully-justified (external) URLS are not messed with, 
      even if they used to resolve to the same host.</small>
    ")
  );
  
  $form['replication']['import_site_prefix'] = array(
    '#type' => 'textfield',
    '#title' => t("Import Site Prefix"),
    '#default_value' => ensure_trailing_slash($profile['import_site_prefix']),
    '#size' => 20,
    '#maxlength' => 100,
    '#description' => t("
      Where, beneath siteroot, should extra imported <strong>pages</strong>
      <em>appear to be</em> served from? 
      This is a prefix that is applied to the generated URL-aliases.
      <br/>
      If you wish to keep the imported
      pages separate from the main site, setting this value to look like a
      subdirectory will make the imported site act like it's in a subsection.
      Otherwise leave it blank and new page URLs will be added under the top
      level navigation.
    "),
  );

  // Even more settings. These ones are able to be overridden during the process also,
  // so they are often slightly more transient preferences
  $form['replication']['subsettings'] = import_html_subsettings_form($profile);

  $form['advanced'] = array(
    '#type' => 'fieldset',
    '#title' => t("Advanced Import Tuning"),
    '#collapsible' => TRUE,
    '#collapsed' => TRUE,
  );
  
  $form['advanced']["help"] = array(
   '#value' => t("How to handle problems when importing"),
  );
  
  $form['advanced']['handle_duplicates'] = array(
    '#type' => 'select',
    '#title' => t("Duplicate Handling"),
    '#default_value' => $profile['handle_duplicates'],
    '#options' => array(IMPORT_HTML_MERGE => 'overwrite/merge' , IMPORT_HTML_SKIP => 'skip'),
    '#description' => t("
      If attempting to import a file into a path that already exists 
      in the system (such as by repeating an import process), do what?
    "),
  );
  
  $form['advanced']['handle_no_title'] = array(
    '#type' => 'select',
    '#title' => t("No Title"),
    '#default_value' => $profile['handle_no_title'],
    '#options' => array(
      IMPORT_HTML_GUESS => 'use the document path',
      IMPORT_HTML_SKIP => 'skip, do not import',
      IMPORT_HTML_DEFAULT => 'set to a placeholder value'
    ),
    '#description' => t("
      If parsing the input document fails to return any title at all, do what?"
    ),
  );

  $form['advanced']['debug_level'] = array(
    '#type' => 'select',
    '#title' => t("Debug Level"),
    '#default_value' => $profile['debug_level'],
    '#options' => array(0, 1, 2, 3),
    '#description' => t("
      To trace and test the import process, increasing the debug level here
      will produce a huge amount of process tracing, 
      including dumps of the raw versions of the input files 
      as it gets massaged into the end result.
      <br/>
      Note that enabling debug can itself introduce errors as the trace logs
      and pretty-printer can cause memory overages.
    "),
  );
  $form['advanced']['keep_temp_files'] = array(
    '#type' => 'checkbox',
    '#title' => t("Keep Temp Files"),
    '#return_value' => TRUE,
    '#default_value' => $profile['keep_temp_files'],
    '#description' => t("
      When processing, files are copied temporarily into a temp directory.
      These are usually deleted immediately after tidying and parsing,
      but if you want to trace problems, enable this option and check the
      files/import directory.
    "),
  );

  
  $form['advanced']['import_html_other_logic'] = array(
    '#value' => t("<p>
       Other logic, like whether to use the TITLE tag or the H1 tag found in a
       file is defined in the XSL template, which you can tune yourself.
       </p>"
    ),
  );

  $form['save'] = array(
    '#type' => 'submit',
    '#value' => t('Save Settings'),
  );
  $form['delete'] = array(
    '#type' => 'submit',
    '#value' => t('Delete Profile'),
  );
  $form['#redirect'] = IMPORT_HTML_ADMIN_PATH .'/profile';

  return $form;
}


/**
 * Returns a widget for selecting an import template from the ones provided by
 * import_html
 */
function import_html_template_selector($current_template) {
    $templates = file_scan_directory(drupal_get_path('module', 'import_html') .'/templates', '.xsl', array('.', '..'), 0, TRUE);
    $options = array_map(
      create_function('$o', 'return $o->name;'),
      $templates
    );
    if(! $current_template) {
      $current_template = drupal_get_path('module', 'import_html') ."/templates/html2simplehtml.xsl";
    }
      
    return array(
      '#type'    => 'select', 
      '#title'   => t('Import template to use'),
      '#options' => $options,
      '#default_value' => $current_template,
      '#prefix'  => '<div id="'. _WRAPPER_IMPORT_TEMPLATE .'-selector" class="sub-option">',
      '#suffix'  => '</div>',
      '#description' => t("
        This file must be a valid XSL Template that can convert from your source
        HTML to the simplified semantic XHTML schema (see docs and examples). 
        Templates should be placed in the import_html module <code>/templates/</code> directory.
        <br/>
        Sample import templates can be found in %module_dir.
      ", array('%module_dir' => drupal_get_path('module', 'import_html') .'/templates' )),
    );
}

/**
 * FAPI callback
 */
function import_html_profile_settings_validate($form_id, $form_values) {
  // Initializing the XSL doc just to check is inefficient, but this is just a config screen.
  if (!_import_html_get_xsl_doc($form_values['translation_template'])) {
    form_set_error('translation_template', t("XSL File %translation_template unable to be parsed", array('%translation_template' => $form_values['translation_template'])));
  }
}

/**
 * FAPI callback
 */
function import_html_profile_settings_submit($form_id, $form_values) {
  if ($form_values['profile_id'] == 'new') {
    drupal_set_message(t('Not saving details called "new"', array()));
    return;
  }
  if ($form_values['op'] == t('Delete Profile')) {
    import_html_profile($form_values['profile_id'], FALSE);
    drupal_set_message(t('%profile_id Profile Deleted', array('%profile_id' => $form_values['profile_id'])));
    return;
  }
  
  // Save only the variables I care about
  $keys = array_keys(import_html_profile_defaults());
  foreach ($keys as $key) {
    $profile[$key] = $form_values[$key];
  }
  // Save it as a blob
  import_html_profile($form_values['profile_id'], $profile);
  drupal_set_message(t('Import HTML %profile_id Profile Updated', array('%profile_id' => $form_values['profile_id'])));
}


/**
 * This is a subset of the general settings - these options are also reproduced
 * on later action pages where they can be tweaked closer to the action. So
 * the form is saved in a block of its own.
 */
function import_html_subsettings_form($profile) {
  $form=array();

  // Taxonomy box:  
  // note, this doesn't show up for vocabs with only one term. Sorta annoying'
  if ($taxonomy = module_invoke('taxonomy', 'form_all', 1)) {
#    $taxonomy[0]='none';
    $form['import_category'] = array(
      '#type' => 'select',
      '#title' => t('Add imported nodes to the following category(s)'),
      '#default_value' => $profile['import_category'],
      '#prefix' => '<div class="criterion">',
      '#size' => 10,
      '#suffix' => '</div>',
      '#options' => $taxonomy,
      '#multiple' => TRUE,
    );
  }
  else {
    $form['import_category'] = array(
      '#value' => t("<p>There are no vocabularies available to tag the imported content with. <a href='!add_vocab'>Create one</a> if you like.</p>", array('!add_vocab' => url('admin/content/taxonomy'))),
    );
  }
  
  // Status selection: 
  $form['import_status'] = array(
    '#type' => 'checkbox',
    '#title' => t('Imported nodes are Published?'),
    '#default_value' => $profile['import_status'],
    '#description' => t("
      Sets the node status.  Check to have nodes published. 
    "),
  );
  
  // promote selection:  added by L0rne
  $form['import_html_import_promote'] = array(
    '#type' => 'checkbox',
    '#title' => t('Imported Nodes are Promoted to front page?'),
    '#default_value' => $profile['import_promote'],
    '#description' => t("
      Sets whether or not imported nodes are promoted to the front page.
    "),
  );
  
  global $user;
  $form['import_user'] = array(
    '#type' => 'textfield', 
    '#title' => t('User to create nodes as'), 
    '#maxlength' => 60, 
    '#autocomplete_path' => 'user/autocomplete', 
    '#default_value' => $profile['import_user'], 
  );

  if (module_exists('menu')) {
    $form["create_menus"] = array(
      '#type' => 'checkbox',
      '#title' => t("Add each page to menu"),
      '#return_value' => TRUE,
      '#default_value' => $profile['create_menus'],
      '#description' => t( 'requires menu.module'. (module_exists('menu')?'(installed)':' (Which is NOT available)')),
    );
    $options = menu_parent_options(0);
    $form["menu_parent_id"] = array(
      '#type' => 'select',
      '#title' => t('Menu Parent'),
      '#default_value' => $profile['menu_parent_id'],
      '#options' => $options,
      '#description' => t("
        Where in the menu system should the new heirachy 
        of pages be built?
        <br/>
        If you leave this as 'Navigation' the new pages 
        may start to collide with your Admin menus, 
        so consider making a placeholder menu first.
      "),
    );
  }
  else {
    $form["create_menus"] = array(
      '#type' => 'markup',
      '#value' => t("
        Menu is not enabled, so the heirachical structure 
        of the imported files will NOT be retained.
      "),
      '#description' => t("
        <p>If you enable the menu.module, 
        the structure of the imported files can be imported as well.</p>
        <p>Note, that in order for this to work, all the apparent parents
        of a page must have menu items as well. If importing a subsection, 
        Placeholders for the higher sections will be created, but they may be 
        hollow shells.</p>
      ")
    );
  }
  return $form;
}  


function import_html_subsettings_submit($form_id, $edit) {
  // Handle incidental settings when submitted from other pages - like during the 
  // import process.

  $profile = import_html_profile($edit['profile_id']);
  $variables = array(
    'import_html_siteroot',
    'import_html_current_subsection',
    'import_site_prefix',
    'import_category',
    'import_status',
    'import_promote',
    'import_user',
    'create_menus',
    'menu_parent_id',
  );
  foreach ($variables as $v) {
   if (isset($edit[$v]) && ($profile[$v] != $edit[$v])) {
    $profile[$v] = $edit[$v];
   }
  }
  
  import_html_profile($edit['profile_id'] , $profile);
  return FALSE;
}

/**
 * Returns a text summary of all the current settings
 */
function import_html_profile_summary($profile) {
  $dest_file_root = url($profile['file_storage_path'], NULL, NULL, TRUE);
  $dest_virtual_path = url($profile['import_site_prefix'], NULL, NULL, TRUE);
  $menu_parent = menu_get_item($profile['menu_parent_id']);
  
  $substitutions = array(
      '%import_html_siteroot' => $profile['siteroot'] ,
      '%import_html_current_subsection' => $profile['current_subsection'],
      '!dest_virtual_path' => l($dest_virtual_path, $dest_virtual_path ),
      '!dest_file_root' => l($dest_file_root, $dest_file_root),
      '!settings_url' => url(IMPORT_HTML_ADMIN_PATH .'/settings'), 
      '!translation_template' => l(basename($profile['translation_template']), $profile['translation_template']), 
      '%import_html_default_document' => $profile['default_document'],
      '%import_html_create_menus' => $profile['create_menus']?'will':'will not',
      '%menu_parent_name' => $menu_parent['title'],
      '%import_html_content_type' => $profile['content_type'],
      '%import_html_import_site_prefix' => $profile['import_site_prefix'],
      '%import_html_import_status' => $profile['import_status']?'will':'will not',
      '%import_html_import_promote' => $profile['import_promote']?'will':'will not',
    );
  
  $summary = array();
  if ($profile['siteroot']) {
    $summary[] = t("
      Source files will be scanned from 
      <br/><code>%import_html_siteroot%import_html_current_subsection</code>", $substitutions);
  }
  $summary[] = t("
    Pages will be re-created underneath
    <br/><code>!dest_virtual_path%import_html_current_subsection</code>
    <br> as <strong>%import_html_content_type</strong> type nodes.", $substitutions);

  $summary[] = t("
    Import template for semantic data extraction is <code>!translation_template</code>
    ", $substitutions);
    
  $summary[] = $profile['rewrite_links'] 
    ? t("Links found within the sources <b>will</b> be rewritten to try and allow for the new paths")
    : t("Links from within the imported pages will <b>not</b> be rewritten, and may get lost.");

  if ($profile['strip_script_tags'])
    $summary[] = t("All script tags found in the source will be discarded");

  $summary[] = t("Non-page files imported will be saved beneath <code>!dest_file_root %import_html_current_subsection</code> ", $substitutions);

  $summary[] = t("Default document <code>%import_html_default_document</code> will be used to represent folders.", $substitutions);

  $summary[] = t("New pages %import_html_create_menus be added to the menu underneath %menu_parent_name.", $substitutions);
  if ($profile['menu_parent_id'] == 1) {
    $summary[] = t("<strong>Warning:</strong> The parent menu is set to the default 'Navigation' menu. This is OK, but will place new pages directly into the top of your menu which can be messy. You may instead wish to create a unique menu holder or menu item and use that as the import menu root instead. 'Primary Links' is also a good choice.", $substitutions);
  }
  $summary[] = t("Imported pages %import_html_import_status be published by default and %import_html_import_promote be promoted to the front page.", $substitutions);

  $output = t("<p>These preferences and more are changeable <a href='!settings_url'>in the settings</a>.</p>", $substitutions);

  $title = t('%profile_label import profile:', array('%profile_label' => $profile['profile_id']));
  return theme('box', $title, theme("item_list", $summary) . $output ); 
}
 
 
///////////////////////////////////////////
// IMPORT PROCESS

/**
 * A multi-part 'wizard' style form.
 * Step 1 - intro and set parameters
 * Step 2 - display files and select them
 * - submit that to run the import
 */
function import_html_process_form($form_values = NULL) {
  // Keep track of the steps
  if (!isset($form_values)) {
    $step = 1;
  }
  else {
    $step = $form_values['step'] + 1;
  }

  $form['step'] = array(
    '#type' => 'hidden',
    '#value' => $step,
  );

  $profile_id = import_html_current_profile_id();  
  $profile = import_html_profile($profile_id);

  switch ($step) {
    case 1:
        // Display current options and patch prior to filesystem scan
        $form['step1'] = import_html_select_source_form($profile);
        $form['summary'] = array('#value' => import_html_profile_summary($profile) ) ;

        if (!module_exists('path')) {
          drupal_set_message(t("The <b>path</b> module is not enabled! If you try to import pages without paths, I can do it, but links won't work and you'll find it hard to find the pages again.'"), 'warning');
        }

      break;

    case 2:
      // Tidy and analyse the data submitted from step 1
      $base_path = ensure_trailing_slash(foreslash($form_values['source_siteroot'])); // win32 safe.
      variable_set('import_html_siteroot', $base_path);
      $form_values['base_path'] = $base_path;
    
      $current_subsection = ensure_trailing_slash($form_values['import_html_current_subsection']);
      $current_subsection = preg_replace('|^/|', '', $current_subsection);
      variable_set('import_html_current_subsection', $current_subsection);
      $form_values['current_subsection'] = $current_subsection;
      
      // Create the file selection form
      // TODO can I avoid doing this file list again when I'm already submitting what I know?
      $form['step2'] = import_html_list_filesystem($form_values, $profile);
      break;
   
    case 3:
      // Files have been selected. They should get added to the queue
#      dpm("import_html_process_form Submitted files, now on step 3. I still need to remake the file list again to ensure they are valid?");
      break;
  }
 
  // This part is important!
  $form['#multistep'] = TRUE;
  $form['#redirect'] = FALSE;

  return $form;
}

/**
 * Handle each step of the import process - by passing the form submit down to per-page handlers
 */
function import_html_process_form_validate($form_id, $form_values) {

  // Massage the input to be forgiving and figure what we really mean
  $base_path = ensure_trailing_slash(foreslash($form_values['source_siteroot'])); // win32 safe.
  $form_values['base_path'] = $base_path;
  $current_subsection = ensure_trailing_slash($form_values['import_html_current_subsection']);
  $current_subsection = preg_replace('|^/|', '', $current_subsection);
  $form_values['current_subsection'] = $current_subsection;

  switch ($form_values['step']) {
    case 1:
      // Check the step 1 submissions
      return import_html_select_source_form_validate($form_id, $form_values);
    case 2:
      // Check the step 2 submissions - all the files have been chosen
      return import_html_list_filesystem_validate($form_id, $form_values);
  }
}

/**
 * Handle each step of the import process - by passing the form submit down to per-page handlers
 */
function import_html_process_form_submit($form_id, $form_values) {
  switch ($form_values['step']) {
    case 1:
      // Process the step 1 submissions (no action just yet)
      # dsm('submitted path selection form');
      break;
    case 2:
      // Received the step 2 submissions - Queue the selected files
      $import_files = $form_values['file_rel_path'];
      // Minor re-hash of the values sent back as checkboxes needed
      $selected_files = array_unique(array_values($import_files));
      unset($selected_files[0]);
      // debug is getting in the way!
      if ($debug_level = import_html_variable('debug_level') && (count($selected_files) > 5) ) {
        drupal_set_message("Temporarily disabling debug level for large imports", 'warning');
        import_html_variable('debug_level', 0);
      }
      switch ($form_values['op']) {
        case t('Queue Files') :
          import_html_queue_files($selected_files, $form_values);
          break;
        case t('Import Files') :
          $results = import_html_import_files($selected_files, $form_values);
          break;
      }
      import_html_variable('debug_level', $debug_level);
      drupal_set_message(t("Imported %count items", array('%count' => count($results))));
      
      break;
    case 3:
      dsm('How did I get here?');
      break;
  }
}


function import_html_select_source_form($profile) {
  $form = array(
  );

  $form['htmlsource'] = array(
    '#type' => 'fieldset',
    '#title' => t("Select HTML file source"),
  );

  $form['htmlsource']['source_siteroot'] = array(
    '#type' => 'textfield',
    '#title' => t("Site Root on the Server"), 
    '#default_value' => variable_get('import_html_siteroot', "/var/www/htdocs/"), 
    '#description' => t("
      Where to read from.
      <br/><b>If the files are on the server</b> (local to Drupal) 
      Enter the absolute or drupal-root-relative location of the site to import. 
      You must have access permissions. 
      Relative paths and aliases will be calculated from here.
      <br/>eg: <code>/var/www/old_site/htdocs</code>, or <code>sites/default/files/copy-of-old-site</code>
    "),
  );

  $form['htmlsource']['import_html_current_subsection'] = array(
    '#type' => 'textfield',
    '#title' => t("Subsection to list"), 
    '#default_value' => variable_get('import_html_current_subsection', '' ),
    '#size' => 20,
    '#description' => t("
      For large sites, its more convenient to just list and process sections.
      If a subdirectory is specified, only that will be displayed."
    ),
  );

  $form[] = array(
    '#type' => 'submit',
    '#value' => t('Next'),
  );

  // Allow user to change some of the big settings here also
  // Set a callback so these settings get saved generically, persistantly
  $form['profile_id'] = array(
    '#type' => 'hidden',
    '#value' => $profile['profile_id'],
  );
  $form['#submit']['import_html_subsettings_submit']=array();
  $form['import_html_settings'] = import_html_subsettings_form($profile);
  $form['import_html_settings']['#type'] = 'fieldset';
  $form['import_html_settings']['#collapsible'] = TRUE;
  $form['import_html_settings']['#collapsed'] = TRUE;
  $form['import_html_settings']['#title'] = t("More Settings");

  return $form; 
}

function import_html_select_source_form_validate($form_id, $form_values) {
  $working_path = $form_values['base_path'] . $form_values['current_subsection'];
  if (! is_dir($working_path)) {
    form_set_error('current_subsection', t("Directory %dir does not exist or is unreadable.", array('%dir' => $working_path)));
    return ;
  }
}


/**
 * Given a local filepath, display all the files I can find in it.
 *
 * Exclude hidden files and directories ( starting with "." or "_" ) .
 * Guess what type of content they are
 * (html, resources like images and styles or other)
 * based on suffix.
 * Enable handy multi-selections to enhance the form
 * 
 * A form api form def used in step 2 of import_html_page()
 * 
 * @param  $base_path
 * 
 * @return Drupal form
 */
function import_html_list_filesystem($form_values, $profile) {

  $base_path = $form_values['base_path'];
  $current_subsection = $form_values['current_subsection'];

  debug("Listing contents of $base_path [$current_subsection]", 2);
  $working_path = $base_path . $current_subsection;
  $form = array();

  // Meh, list everything and filter them using user prefs  
  $dir_structure = file_scan_directory(trim_trailing_slash($working_path), ".*");
  #debug_pre($dir_structure, 4);

  // file_scan_directory returns a flat array. Convert it into a tree structure, then render it.
  $tree = import_html_sort_list_into_tree($dir_structure, $base_path, $profile['file_exclusions']);

  // note this info in the form for the rendering function to display later
  $form['file_count'] = array( '#type' => 'value', '#value' => count($dir_structure));
  $form['current_subsection'] = array( '#type' => 'value', '#value' => $current_subsection);

  // Note the current context as the submit process needs to know
  $form['base_path'] = array( '#type' => 'value', '#value' => $form_values['base_path']);
  $form['current_subsection'] = array( '#type' => 'value', '#value' => $form_values['current_subsection']);
  $form['profile_id'] = array('#type' => 'value', '#value' => $form_values['profile_id']);

  $dest_file_root = url($profile['file_storage_path'], NULL, NULL, TRUE);
  $dest_virtual_path = url($profile['import_site_prefix'], NULL, NULL, TRUE);

  // UI thingie. This is a placeholder. 
  // Actual functionality gets via js added post-load. 
  // @see filetype_selectors.js
  $form['selectors'] = array(
    '#weight' => -2,
    '#value' => "<p id='import-html-selectors'>Select Files to import</p>",
  );  


  // Need to convert the sructured tree into a structured form
  $tree_form = _import_html_tree_to_form($tree, "", "/");
  $form['filesystem'] = $tree_form;
  $form['#theme'] = 'import_html_list_filesystem';

  $form['action'] = array(
    '#value' => '<p>'. t("Either run the import now, or queue the selected files for processing later - may be required for large jobs") .'</p>',
  );
  $form['queue'] = array(
    '#type' => 'submit',
    '#value' => t('Queue Files'),
  );
  $form['import'] = array(
    '#type' => 'submit',
    '#value' => t('Import Files'),
  );
  
  return $form;
}


function import_html_list_filesystem_validate($form_id, $form_values) {
  $import_files = $form_values['file_rel_path'];
  // Minor re-hash of the values sent back as checkboxes needed
  $selected_files = array_unique(array_values($import_files));
  unset($selected_files[0]);

  if ( empty($selected_files)) {
    form_set_error('file_rel_path', t("No Files Selected. Nothing to import"));
    return;
  }
}

/**
 * From a flat list of file defs, build a tree structure, and annotate it like
 * the forms API does.
 * todo DEPRECATE and include the formtree extension
 */
function import_html_sort_list_into_tree($dir_structure, $base_path, $file_exclusions = "^_\nCVS\n^\\.") {

  $ex = trim(preg_replace('|\r|', "", $file_exclusions));
  $exclusions = explode("\n", $ex);

  $tree = array();
  $count = 0;
  foreach ($dir_structure as $file_path => $file_info) {
    $rel_path = substr($file_path, strlen($base_path));

    // skip if it's a hidden/excluded file
    foreach ($exclusions as $regexp) {
      if (preg_match('|'. $regexp .'|', $rel_path)) { continue 2; }
    }

    $ancestors = preg_split("/[\/\:]+/", $rel_path);
    $twig = & $tree; // find the current twig, starting from the top
    $path = '';
    while ($dad = array_shift($ancestors)) {
      // walk down the path
      if (!array_key_exists($dad, $twig)) {
        // add new branch
        $twig['#type'] = 'disk_folder';
        $twig['#value']= $path;
        $twig['#description']= basename($path);
        $twig['#filename']   = $base_path . $path; 

        $twig[$dad] = array();
        $count++;
      }
      $path .= $dad .'/';
      $twig = & $twig[$dad];
    }
    $twig['#type']       = 'disk_file';
    $twig['#description']= $file_info->basename;
    $twig['#filename']   = $file_path; 
    $twig['#value']      = $rel_path;
  }
  return $tree;  
}


/**
 * Recursive form construction function to format a tree-like structure of heirarchical items
 *
 * The function is self-referentially recursive - depth-first formatting of a tree menu.
 *
 * @param $tree array The tree layout so far;
 * @param $name string The human name of the item were are adding right now
 * @param $prefix string The path so far;
 * @param $url string The full address of the item we are adding now
 * 
 * @return form object
 */
function _import_html_tree_to_form($tree, $name, $prefix = "", $url = "?") {
  $path = (($prefix != "/") ? $prefix ."/" : "") . $name;
  $id = "ID_" . preg_replace("/[^\w]/", "", $path);
  // $id is a safe name - a unique id derived from the item path
  // -- OK, these are becoming stupidly long - any reason not to just use a counter?
  
  // To sorta-but-not-really 'tree' and allow multiple values for identically named checkboxes,
  // we keep a running list of numbers. FAPI doesn't read in multiples unless we told it to.
  // We cheat by setting the '#parents' manually before rendering, and using a counter 
  static $checkbox_count;

  // special case:  if name is blank (we are at root) show the working dir
  if (! $tree['#value']) $tree['#value'] = "<big>". variable_get('import_html_siteroot', "") ."</big>";

  $element = array();
  $element['#return_value'] = $tree['#value'];
  $item_type = ($tree['#type'] == 'disk_folder') ? "container" : import_html_guess_file_class($tree['#filename']);
  $element['#item-type'] = $item_type; // made-up attribute to pass to theme
  $element['#item-path'] = $item_type; // made-up attribute to pass to theme
  $element['#attributes']['class'] = $item_type;
  
  if ($tree['#type'] == 'disk_folder') {
    $element['#type'] = 'fieldset';
    $element['#title'] = $tree['#value'];
#    $element['#id'] = "${id}-container";
    $element['#attributes']['class'] = "tree-branch";
    $element['#theme'] = 'import_html_filesystem_tree_container';
    // checkbox toggle for the children group
    $element['file_rel_dir'] = array(
     '#type' => 'checkbox',
     '#title' => $tree['#value'],
     '#id' => "${id}-toggle",
     '#item-type' => 'container',
     '#return-value' => $tree['#value'],
     // Need to allow multiples, but they are essentially flat, not tree-d
     '#parents' => array('file_rel_dir', $checkbox_count++),
    );
    $element['file_rel_dir']['#attributes']['class'] = "tree-branch-toggle";
  } 
  else {
    $element['#type'] = 'checkbox';
    $element['#title'] = $tree['#description'];
    $element['#theme'] = 'import_html_filesystem_tree_item';
    $element['#id'] = "${id}-checkbox";
    $element['#attributes']['class'] = "tree-leaf";

    // Need to allow multiples, but they are essentially flat, not tree-d
    $element['#parents'] = array('file_rel_path', $checkbox_count++);
  }
  
  // Construct the content

  if (element_children($tree)) {
    // This node has content to be recursed into. Add it as a child
    ksort($tree);
    $element['children'] = array();
    // Just a container array. Themed later
    $element['children']['#id'] = "${id}-content";
    $element['children']['#attributes']['class'] = "tree-content";
    foreach ($tree as $twigname => $twig) {
      if (element_property($twigname)) {continue;}
      $element['children']['file_rel_path'][] = _import_html_tree_to_form($twig, $twigname, $path);
    }
  }
  return $element;
}

/**
 * should put the selectors magic here?
 */
function theme_import_html_list_filesystem($element) {
  drupal_add_js(drupal_get_path('module', 'import_html') .'/toggle_treeview.js');
  drupal_add_css(drupal_get_path('module', 'import_html') .'/treeview.css');
  drupal_add_js(drupal_get_path('module', 'import_html') .'/filetype_selectors.js');
  return drupal_render($element);
}

/**
 * Basically the theme_checkbox, but without form-item div cruft
 */
function theme_import_html_filesystem_tree_item($element) {
  $checked = $element['#value'] ? ' checked="checked" ' : ' ';
  $atts = drupal_attributes($element['#attributes']);
  $checkbox = "<input type='checkbox' name='{$element['#name']}' id='{$element['#id']}' value='{$element['#return_value']}' $checked $atts />";

  $label_atts = array(
    'class' => $element['#item-type'] .'-item file-label',
    'title' => $element['#return_value'],
    'for' => $element['#id'],
  );
  if ($element['#item-type'] == 'container') {
    $label_atts['class'] = 'tree-branch-label';
  }

  $checkbox = '<label '. drupal_attributes($label_atts) .'>'. $checkbox .' '. $element['#title'] .'</label>';
  return $checkbox;
}

/**
 * filesystem tree container is a fieldset, contains one toggle checkbox and a collection of 'children'
 * If you want the normal rendering back (description, value) put it in yourself.
 */
function theme_import_html_filesystem_tree_container($element) {
  $output = theme('import_html_filesystem_tree_item', $element['file_rel_dir'] ); // the whole-group checkbox
  $output .= 
    '<div'. drupal_attributes($element['children']['#attributes']) .'>' 
    . drupal_render($element['children'] )
    .'</div>'
    ;

  return $output;
}


/////////////////////////////////////////////
// DEMO
/////////////////////////////////////////////


/**
 * Show an interface to import just one file
 */
function import_html_demo_form() {
  $form = array();

  $form = array(
   '#method' => 'post',
  );

  $form['Select'] = array(
    '#type' => 'fieldset',
    '#title' => t("Select HTML file source"),
  );
  $form['Select']['source_url'] = array(
    '#type' => 'textfield',
    '#title' => t("URL to import"),
    '#default_value' => drupal_get_path('module', 'import_html') . '/templates/input_example_03.htm',
    '#size' => 80,