plugins/FeedsNodeProcessor.inc

   1 <?php
   2 // $Id$
   3
   4 /**
   5  * @file
   6  * Class definition of FeedsNodeProcessor.
   7  */
   8
   9 // Create or delete FEEDS_NODE_BATCH_SIZE at a time.
  10 define('FEEDS_NODE_BATCH_SIZE', 50);
  11
  12 // Deprecated. Use FEEDS_SKIPE_EXISTING, FEEDS_REPLACE_EXISTNG,
  13 // FEEDS_UPDATE_EXISTING instead.
  14 define('FEEDS_NODE_SKIP_EXISTING', 0);
  15 define('FEEDS_NODE_REPLACE_EXISTING', 1);
  16 define('FEEDS_NODE_UPDATE_EXISTING', 2);
  17
  18 // "Use the present default format"
  19 define('FEEDS_NODE_DEFAULT_FORMAT', -1);
  20
  21 /**
  22  * Creates nodes from feed items.
  23  */
  24 class FeedsNodeProcessor extends FeedsProcessor {
  25
  26   /**
  27    * Implements FeedsProcessor::process().
  28    */
  29   public function process(FeedsImportBatch $batch, FeedsSource $source) {
  30
  31     // Keep track of processed items in this pass, set total number of items.
  32     $processed = 0;
  33     if (!$batch->getTotal(FEEDS_PROCESSING)) {
  34       $batch->setTotal(FEEDS_PROCESSING, $batch->getItemCount());
  35     }
  36
  37     while ($item = $batch->shiftItem()) {
  38
  39       // Create/update if item does not exist or update existing is enabled.
  40       if (!($nid = $this->existingItemId($batch, $source)) || ($this->config['update_existing'] != FEEDS_SKIP_EXISTING)) {
  41         // Only proceed if item has actually changed.
  42         $hash = $this->hash($item);
  43         if (!empty($nid) && $hash == $this->getHash($nid)) {
  44           continue;
  45         }
  46
  47         $node = $this->buildNode($nid, $source->feed_nid);
  48         $node->feeds_node_item->hash = $hash;
  49
  50         // Map and save node. If errors occur don't stop but report them.
  51         try {
  52           $this->map($batch, $node);
  53           node_save($node);
  54           if (!empty($nid)) {
  55             $batch->updated++;
  56           }
  57           else {
  58             $batch->created++;
  59           }
  60         }
  61         catch (Exception $e) {
  62           drupal_set_message($e->getMessage(), 'warning');
  63           watchdog('feeds', $e->getMessage(), array(), WATCHDOG_WARNING);
  64         }
  65       }
  66
  67       $processed++;
  68       if ($processed >= variable_get('feeds_node_batch_size', FEEDS_NODE_BATCH_SIZE)) {
  69         $batch->setProgress(FEEDS_PROCESSING, $batch->created + $batch->updated);
  70         return;
  71       }
  72     }
  73
  74     // Set messages.
  75     if ($batch->created) {
  76       drupal_set_message(format_plural($batch->created, 'Created @number @type node.', 'Created @number @type nodes.', array('@number' => $batch->created, '@type' => node_type_get_name($this->config['content_type']))));
  77     }
  78     elseif ($batch->updated) {
  79       drupal_set_message(format_plural($batch->updated, 'Updated @number @type node.', 'Updated @number @type nodes.', array('@number' => $batch->updated, '@type' => node_type_get_name($this->config['content_type']))));
  80     }
  81     else {
  82       drupal_set_message(t('There is no new content.'));
  83     }
  84     $batch->setProgress(FEEDS_PROCESSING, FEEDS_BATCH_COMPLETE);
  85   }
  86
  87   /**
  88    * Implements FeedsProcessor::clear().
  89    */
  90   public function clear(FeedsBatch $batch, FeedsSource $source) {
  91     if (!$batch->getTotal(FEEDS_CLEARING)) {
  92       $total = db_query("SELECT COUNT(n.nid) FROM {node} n JOIN {feeds_node_item} fn ON n.nid = fn.nid WHERE fn.id = :id AND fn.feed_nid = :nid", array(':id' => $source->id, ':nid' => $source->feed_nid))->fetchField();
  93       $batch->setTotal(FEEDS_CLEARING, $total);
  94     }
  95     $count = variable_get('feeds_node_batch_size', FEEDS_NODE_BATCH_SIZE);
  96     $nids = array();
  97     $nodes = db_query_range("SELECT n.nid FROM {node} n JOIN {feeds_node_item} fn ON n.nid = fn.nid WHERE fn.id = :id AND fn.feed_nid = :nid", 0, $count, array(':id' => $source->id, ':nid' => $source->feed_nid));
  98     foreach ($nodes as $node) {
  99       $nids[$node->nid] = $node->nid;
 100       $batch->deleted++;
 101     }
 102     node_delete_multiple($nids);
 103     if (db_query_range("SELECT 1 FROM {node} n JOIN {feeds_node_item} fn ON n.nid = fn.nid WHERE fn.id = :id AND fn.feed_nid = :nid", 0, 1, array(':id' => $source->id, ':nid' => $source->feed_nid))->fetchField()) {
 104       $batch->setProgress(FEEDS_CLEARING, $batch->deleted);
 105       return;
 106     }
 107
 108     // Set message.
 109     drupal_get_messages('status');
 110     if ($batch->deleted) {
 111       drupal_set_message(format_plural($batch->deleted, 'Deleted @number node.', 'Deleted @number nodes.', array('@number' => $batch->deleted)));
 112     }
 113     else {
 114       drupal_set_message(t('There is no content to be deleted.'));
 115     }
 116     $batch->setProgress(FEEDS_CLEARING, FEEDS_BATCH_COMPLETE);
 117   }
 118
 119   /**
 120    * Implement expire().
 121    */
 122   public function expire($time = NULL) {
 123     if ($time === NULL) {
 124       $time = $this->expiryTime();
 125     }
 126     if ($time == FEEDS_EXPIRE_NEVER) {
 127       return;
 128     }
 129     $count = variable_get('feeds_node_batch_size', FEEDS_NODE_BATCH_SIZE);
 130     $nodes = db_query_range("SELECT n.nid FROM {node} n JOIN {feeds_node_item} fni ON n.nid = fni.nid WHERE fni.id = :id AND n.created < :created", 0, $count, array(':id' => $this->id, ':created' => REQUEST_TIME - $time));
 131     foreach ($nodes as $node) {
 132       _feeds_node_delete($node->nid);
 133     }
 134     if (db_query_range("SELECT 1 FROM {node} n JOIN {feeds_node_item} fni ON n.nid = fni.nid WHERE fni.id = :id AND n.created < :created", 0, 1, array(':id' => $this->id, ':created' => REQUEST_TIME - $time))->fetchField()) {
 135       return FEEDS_BATCH_ACTIVE;
 136     }
 137     return FEEDS_BATCH_COMPLETE;
 138   }
 139
 140   /**
 141    * Return expiry time.
 142    */
 143   public function expiryTime() {
 144     return $this->config['expire'];
 145   }
 146
 147   /**
 148    * Override parent::configDefaults().
 149    */
 150   public function configDefaults() {
 151     $types = node_type_get_names();
 152     $type = isset($types['article']) ? 'article' : key($types);
 153     return array(
 154       'content_type' => $type,
 155       'input_format' => FEEDS_NODE_DEFAULT_FORMAT,
 156       'update_existing' => FEEDS_SKIP_EXISTING,
 157       'expire' => FEEDS_EXPIRE_NEVER,
 158       'mappings' => array(),
 159       'author' => 0,
 160     );
 161   }
 162
 163   /**
 164    * Override parent::configForm().
 165    */
 166   public function configForm(&$form_state) {
 167     $types = node_type_get_names();
 168     array_walk($types, 'check_plain');
 169     $form = array();
 170     $form['content_type'] = array(
 171       '#type' => 'select',
 172       '#title' => t('Content type'),
 173       '#description' => t('Select the content type for the nodes to be created. <strong>Note:</strong> Users with "import !feed_id feeds" permissions will be able to <strong>import</strong> nodes of the content type selected here regardless of the node level permissions. Further, users with "clear !feed_id permissions" will be able to <strong>delete</strong> imported nodes regardless of their node level permissions.', array('!feed_id' => $this->id)),
 174       '#options' => $types,
 175       '#default_value' => $this->config['content_type'],
 176     );
 177     $format_options = array(FEEDS_NODE_DEFAULT_FORMAT => t('Default format'));
 178     global $user;
 179     $formats = filter_formats($user);
 180       foreach ($formats as $format) {
 181         $format_options[$format->format] = check_plain($format->name);
 182       }
 183     $form['input_format'] = array(
 184       '#type' => 'select',
 185       '#title' => t('Input format'),
 186       '#description' => t('Select the input format for the body field of the nodes to be created.'),
 187       '#options' => $format_options,
 188       '#default_value' => $this->config['input_format'],
 189     );
 190     $author = user_load($this->config['author']);
 191     $form['author'] = array(
 192       '#type' => 'textfield',
 193       '#title' => t('Author'),
 194       '#description' => t('Select the author of the nodes to be created - leave empty to assign "anonymous".'),
 195       '#autocomplete_path' => 'user/autocomplete',
 196       '#default_value' => empty($author->name) ?  'anonymous' : check_plain($author->name),
 197     );
 198     $period = drupal_map_assoc(array(FEEDS_EXPIRE_NEVER, 3600, 10800, 21600, 43200, 86400, 259200, 604800, 604800 * 4, 604800 * 12, 604800 * 24, 31536000), 'feeds_format_expire');
 199     $form['expire'] = array(
 200       '#type' => 'select',
 201       '#title' => t('Expire nodes'),
 202       '#options' => $period,
 203       '#description' => t('Select after how much time nodes should be deleted. The node\'s published date will be used for determining the node\'s age, see Mapping settings.'),
 204       '#default_value' => $this->config['expire'],
 205     );
 206     $form['update_existing'] = array(
 207       '#type' => 'radios',
 208       '#title' => t('Update existing nodes'),
 209       '#description' => t('Select how existing nodes should be updated. Existing nodes will be determined using mappings that are a "unique target".'),
 210       '#options' => array(
 211         FEEDS_SKIP_EXISTING => 'Do not update existing nodes',
 212         FEEDS_REPLACE_EXISTING => 'Replace existing nodes',
 213         FEEDS_UPDATE_EXISTING => 'Update existing nodes (slower than replacing them)',
 214       ),
 215       '#default_value' => $this->config['update_existing'],
 216     );
 217     return $form;
 218   }
 219
 220   /**
 221    * Override parent::configFormValidate().
 222    */
 223   public function configFormValidate(&$values) {
 224     if ($author = user_load_by_name($values['author'])) {
 225       $values['author'] = $author->uid;
 226     }
 227     else {
 228       $values['author'] = 0;
 229     }
 230   }
 231
 232   /**
 233    * Reschedule if expiry time changes.
 234    */
 235   public function configFormSubmit(&$values) {
 236     if ($this->config['expire'] != $values['expire']) {
 237       feeds_reschedule($this->id);
 238     }
 239     parent::configFormSubmit($values);
 240   }
 241
 242   /**
 243    * Override setTargetElement to operate on a target item that is a node.
 244    */
 245   public function setTargetElement($target_node, $target_element, $value) {
 246     if (in_array($target_element, array('url', 'guid'))) {
 247       $target_node->feeds_node_item->$target_element = $value;
 248     }
 249     elseif (in_array($target_element, array('title', 'status', 'created', 'nid', 'uid'))) {
 250       $target_node->$target_element = $value;
 251     }
 252   }
 253
 254   /**
 255    * Return available mapping targets.
 256    */
 257   public function getMappingTargets() {
 258     $type = node_type_get_type($this->config['content_type']);
 259     $targets = array();
 260     if ($type->has_title) {
 261       $targets['title'] = array(
 262         'name' => t('Title'),
 263         'description' => t('The title of the node.'),
 264       );
 265     }
 266     $targets += array(
 267       'nid' => array(
 268         'name' => t('Node ID'),
 269         'description' => t('The nid of the node. NOTE: use this feature with care, node ids are usually assigned by Drupal.'),
 270         'optional_unique' => TRUE,
 271       ),
 272       'uid' => array(
 273         'name' => t('User ID'),
 274         'description' => t('The Drupal user ID of the node author.'),
 275       ),
 276       'status' => array(
 277         'name' => t('Published status'),
 278         'description' => t('Whether a node is published or not. 1 stands for published, 0 for not published.'),
 279       ),
 280       'created' => array(
 281         'name' => t('Published date'),
 282         'description' => t('The UNIX time when a node has been published.'),
 283       ),
 284       'url' => array(
 285         'name' => t('URL'),
 286         'description' => t('The external URL of the node. E. g. the feed item URL in the case of a syndication feed. May be unique.'),
 287         'optional_unique' => TRUE,
 288       ),
 289       'guid' => array(
 290         'name' => t('GUID'),
 291         'description' => t('The external GUID of the node. E. g. the feed item GUID in the case of a syndication feed. May be unique.'),
 292         'optional_unique' => TRUE,
 293       ),
 294     );
 295
 296     // Let other modules expose mapping targets.
 297     self::loadMappers();
 298     feeds_alter('feeds_processor_targets', $targets, 'node', $this->config['content_type']);
 299
 300     return $targets;
 301   }
 302
 303   /**
 304    * Get nid of an existing feed item node if available.
 305    */
 306   protected function existingItemId(FeedsImportBatch $batch, FeedsSource $source) {
 307
 308     // Iterate through all unique targets and test whether they do already
 309     // exist in the database.
 310     foreach ($this->uniqueTargets($batch) as $target => $value) {
 311       switch ($target) {
 312         case 'nid':
 313           $nid = db_query("SELECT nid FROM {node} WHERE nid = :nid", array(':nid' => $value))->fetchField();
 314           break;
 315         case 'url':
 316           $nid = db_query("SELECT nid FROM {feeds_node_item} WHERE feed_nid = :nid AND id = :id AND url = :url", array(':nid' => $source->feed_nid, ':id' => $source->id, ':url' => $value))->fetchField();
 317           break;
 318         case 'guid':
 319           $nid = db_query("SELECT nid FROM {feeds_node_item} WHERE feed_nid = :nid AND id = :id AND guid = :guid", array(':nid' => $source->feed_nid, ':id' => $source->id, ':guid' => $value))->fetchField();
 320           break;
 321       }
 322       if ($nid) {
 323         // Return with the first nid found.
 324         return $nid;
 325       }
 326     }
 327     return 0;
 328   }
 329
 330   /**
 331    * Creates a new node object in memory and returns it.
 332    */
 333   protected function buildNode($nid, $feed_nid) {
 334     $node = new stdClass();
 335     if (empty($nid)) {
 336       $node->created = REQUEST_TIME;
 337       $populate = TRUE;
 338     }
 339     else {
 340       if ($this->config['update_existing'] == FEEDS_UPDATE_EXISTING) {
 341         $node = node_load($nid, NULL, TRUE);
 342       }
 343       else {
 344         $node->nid = $nid;
 345         $node->vid = db_query("SELECT vid FROM {node} WHERE nid = :nid", array(':nid' => $nid))->fetchField();
 346         $populate = TRUE;
 347       }
 348     }
 349     if ($populate) {
 350       $node->type = $this->config['content_type'];
 351       $node->changed = REQUEST_TIME;
 352       $node->format = ($this->config['input_format'] == FEEDS_NODE_DEFAULT_FORMAT) ? filter_fallback_format() : $this->config['input_format'];
 353       $node->feeds_node_item = new stdClass();
 354       $node->feeds_node_item->id = $this->id;
 355       $node->feeds_node_item->imported = REQUEST_TIME;
 356       $node->feeds_node_item->feed_nid = $feed_nid;
 357       $node->feeds_node_item->url = '';
 358       $node->feeds_node_item->guid = '';
 359     }
 360
 361     // Give mappers a hint at what they're operating on.
 362     $node->entity_type = 'node';
 363
 364     // Let other modules populate default values.
 365     node_object_prepare($node);
 366
 367     // Populate properties that are set by node_object_prepare().
 368     $node->log = 'Created/updated by FeedsNodeProcessor';
 369     if ($populate) {
 370       $node->uid = $this->config['author'];
 371     }
 372     return $node;
 373   }
 374
 375   /**
 376    * Create MD5 hash of item and mappings array.
 377    *
 378    * Include mappings as a change in mappings may have an affect on the item
 379    * produced.
 380    *
 381    * @return Always returns a hash, even with empty, NULL, FALSE:
 382    *  Empty arrays return 40cd750bba9870f18aada2478b24840a
 383    *  Empty/NULL/FALSE strings return d41d8cd98f00b204e9800998ecf8427e
 384    */
 385   protected function hash($item) {
 386     static $serialized_mappings;
 387     if (!$serialized_mappings) {
 388       $serialized_mappings = serialize($this->config['mappings']);
 389     }
 390     return hash('md5', serialize($item) . $serialized_mappings);
 391   }
 392
 393   /**
 394    * Retrieve MD5 hash of $nid from DB.
 395    * @return Empty string if no item is found, hash otherwise.
 396    */
 397   protected function getHash($nid) {
 398     if ($hash = db_query("SELECT hash FROM {feeds_node_item} WHERE nid = :nid", array(':nid' => $nid))->fetchField()) {
 399       // Return with the hash.
 400       return $hash;
 401     }
 402     return '';
 403   }
 404 }