Update to Drupal 8.2.0. For more information, see https://www.drupal.org/project/drupal/releases/8.2.0

This commit is contained in:
Pantheon Automation 2016-10-06 15:16:20 -07:00 committed by Greg Anderson
parent 2f563ab520
commit f1c8716f57
1732 changed files with 52334 additions and 11780 deletions

View file

@ -24,7 +24,7 @@ use Drupal\Component\Annotation\Plugin;
*
* @Annotation
*/
class MigrateSource extends Plugin {
class MigrateSource extends Plugin implements MultipleProviderAnnotationInterface {
/**
* A unique identifier for the process plugin.
@ -66,4 +66,34 @@ class MigrateSource extends Plugin {
*/
public $minimum_version;
/**
* {@inheritdoc}
*/
public function getProvider() {
if (isset($this->definition['provider'])) {
return is_array($this->definition['provider']) ? reset($this->definition['provider']) : $this->definition['provider'];
}
return FALSE;
}
/**
* {@inheritdoc}
*/
public function getProviders() {
if (isset($this->definition['provider'])) {
// Ensure that we return an array even if
// \Drupal\Component\Annotation\AnnotationInterface::setProvider() has
// been called.
return (array) $this->definition['provider'];
}
return [];
}
/**
* {@inheritdoc}
*/
public function setProviders(array $providers) {
$this->definition['provider'] = $providers;
}
}

View file

@ -0,0 +1,41 @@
<?php
namespace Drupal\migrate\Annotation;
use Drupal\Component\Annotation\AnnotationInterface;
/**
* Defines a common interface for classed annotations with multiple providers.
*
* @todo This is a temporary solution to the fact that migration source plugins
* have more than one provider. This functionality will be moved to core in
* https://www.drupal.org/node/2786355.
*/
interface MultipleProviderAnnotationInterface extends AnnotationInterface {
/**
* Gets the name of the provider of the annotated class.
*
* @return string
* The provider of the annotation. If there are multiple providers the first
* is returned.
*/
public function getProvider();
/**
* Gets the provider names of the annotated class.
*
* @return string[]
* The providers of the annotation.
*/
public function getProviders();
/**
* Sets the provider names of the annotated class.
*
* @param string[] $providers
* The providers of the annotation.
*/
public function setProviders(array $providers);
}

View file

@ -0,0 +1,60 @@
<?php
namespace Drupal\migrate\Event;
use Drupal\migrate\Plugin\MigrationInterface;
use Drupal\migrate\MigrateMessageInterface;
use Symfony\Component\EventDispatcher\Event as SymfonyEvent;
class EventBase extends SymfonyEvent {
/**
* The migration.
*
* @var \Drupal\migrate\Plugin\MigrationInterface
*/
protected $migration;
/**
* The current message service.
*
* @var \Drupal\migrate\MigrateMessageInterface
*/
protected $message;
/**
* Constructs a Migrate event object.
*
* @param \Drupal\migrate\Plugin\MigrationInterface $migration
* The migration being run.
* @param \Drupal\migrate\MigrateMessageInterface $message
* The Migrate message service.
*/
public function __construct(MigrationInterface $migration, MigrateMessageInterface $message) {
$this->migration = $migration;
$this->message = $message;
}
/**
* Gets the migration.
*
* @return \Drupal\migrate\Plugin\MigrationInterface
* The migration being run.
*/
public function getMigration() {
return $this->migration;
}
/**
* Logs a message using the Migrate message service.
*
* @param string $message
* The message to log.
* @param string $type
* The type of message, for example: status or warning.
*/
public function logMessage($message, $type = 'status') {
$this->message->display($message, $type);
}
}

View file

@ -0,0 +1,26 @@
<?php
namespace Drupal\migrate\Event;
/**
* Interface for plugins that react to pre- or post-import events.
*/
interface ImportAwareInterface {
/**
* Performs pre-import tasks.
*
* @param \Drupal\migrate\Event\MigrateImportEvent $event
* The pre-import event object.
*/
public function preImport(MigrateImportEvent $event);
/**
* Performs post-import tasks.
*
* @param \Drupal\migrate\Event\MigrateImportEvent $event
* The post-import event object.
*/
public function postImport(MigrateImportEvent $event);
}

View file

@ -2,39 +2,7 @@
namespace Drupal\migrate\Event;
use Drupal\migrate\Plugin\MigrationInterface;
use Symfony\Component\EventDispatcher\Event;
/**
* Wraps a pre- or post-import event for event listeners.
*/
class MigrateImportEvent extends Event {
/**
* Migration entity.
*
* @var \Drupal\migrate\Plugin\MigrationInterface
*/
protected $migration;
/**
* Constructs an import event object.
*
* @param \Drupal\migrate\Plugin\MigrationInterface $migration
* Migration entity.
*/
public function __construct(MigrationInterface $migration) {
$this->migration = $migration;
}
/**
* Gets the migration entity.
*
* @return \Drupal\migrate\Plugin\MigrationInterface
* The migration entity involved.
*/
public function getMigration() {
return $this->migration;
}
}
class MigrateImportEvent extends EventBase {}

View file

@ -3,6 +3,7 @@
namespace Drupal\migrate\Event;
use Drupal\migrate\Plugin\MigrationInterface;
use Drupal\migrate\MigrateMessageInterface;
use Drupal\migrate\Row;
/**
@ -10,18 +11,27 @@ use Drupal\migrate\Row;
*/
class MigratePostRowSaveEvent extends MigratePreRowSaveEvent {
/**
* The row's destination ID.
*
* @var array|bool
*/
protected $destinationIdValues = [];
/**
* Constructs a post-save event object.
*
* @param \Drupal\migrate\Plugin\MigrationInterface $migration
* Migration entity.
* @param \Drupal\migrate\MigrateMessageInterface $message
* The message interface.
* @param \Drupal\migrate\Row $row
* Row object.
* @param array|bool $destination_id_values
* Values represent the destination ID.
*/
public function __construct(MigrationInterface $migration, Row $row, $destination_id_values) {
parent::__construct($migration, $row);
public function __construct(MigrationInterface $migration, MigrateMessageInterface $message, Row $row, $destination_id_values) {
parent::__construct($migration, $message, $row);
$this->destinationIdValues = $destination_id_values;
}

View file

@ -3,13 +3,13 @@
namespace Drupal\migrate\Event;
use Drupal\migrate\Plugin\MigrationInterface;
use Drupal\migrate\MigrateMessageInterface;
use Drupal\migrate\Row;
use Symfony\Component\EventDispatcher\Event;
/**
* Wraps a pre-save event for event listeners.
*/
class MigratePreRowSaveEvent extends Event {
class MigratePreRowSaveEvent extends EventBase {
/**
* Row object.
@ -18,34 +18,20 @@ class MigratePreRowSaveEvent extends Event {
*/
protected $row;
/**
* Migration entity.
*
* @var \Drupal\migrate\Plugin\MigrationInterface
*/
protected $migration;
/**
* Constructs a pre-save event object.
*
* @param \Drupal\migrate\Plugin\MigrationInterface $migration
* Migration entity.
* @param \Drupal\migrate\MigrateMessageInterface $message
* The current migrate message service.
* @param \Drupal\migrate\Row $row
*/
public function __construct(MigrationInterface $migration, Row $row) {
$this->migration = $migration;
public function __construct(MigrationInterface $migration, MigrateMessageInterface $message, Row $row) {
parent::__construct($migration, $message);
$this->row = $row;
}
/**
* Gets the migration entity.
*
* @return \Drupal\migrate\Plugin\MigrationInterface
* The migration entity being imported.
*/
public function getMigration() {
return $this->migration;
}
/**
* Gets the row object.
*

View file

@ -0,0 +1,26 @@
<?php
namespace Drupal\migrate\Event;
/**
* Interface for plugins that react to pre- or post-rollback events.
*/
interface RollbackAwareInterface {
/**
* Performs pre-rollback tasks.
*
* @param \Drupal\migrate\Event\MigrateRollbackEvent $event
* The pre-rollback event object.
*/
public function preRollback(MigrateRollbackEvent $event);
/**
* Performs post-rollback tasks.
*
* @param \Drupal\migrate\Event\MigrateRollbackEvent $event
* The post-rollback event object.
*/
public function postRollback(MigrateRollbackEvent $event);
}

View file

@ -177,7 +177,7 @@ class MigrateExecutable implements MigrateExecutableInterface {
)), 'error');
return MigrationInterface::RESULT_FAILED;
}
$this->getEventDispatcher()->dispatch(MigrateEvents::PRE_IMPORT, new MigrateImportEvent($this->migration));
$this->getEventDispatcher()->dispatch(MigrateEvents::PRE_IMPORT, new MigrateImportEvent($this->migration, $this->message));
// Knock off migration if the requirements haven't been met.
try {
@ -185,11 +185,17 @@ class MigrateExecutable implements MigrateExecutableInterface {
}
catch (RequirementsException $e) {
$this->message->display(
$this->t('Migration @id did not meet the requirements. @message @requirements', array(
'@id' => $this->migration->id(),
'@message' => $e->getMessage(),
'@requirements' => $e->getRequirementsString(),
)), 'error');
$this->t(
'Migration @id did not meet the requirements. @message @requirements',
array(
'@id' => $this->migration->id(),
'@message' => $e->getMessage(),
'@requirements' => $e->getRequirementsString(),
)
),
'error'
);
return MigrationInterface::RESULT_FAILED;
}
@ -229,9 +235,9 @@ class MigrateExecutable implements MigrateExecutableInterface {
if ($save) {
try {
$this->getEventDispatcher()->dispatch(MigrateEvents::PRE_ROW_SAVE, new MigratePreRowSaveEvent($this->migration, $row));
$this->getEventDispatcher()->dispatch(MigrateEvents::PRE_ROW_SAVE, new MigratePreRowSaveEvent($this->migration, $this->message, $row));
$destination_id_values = $destination->import($row, $id_map->lookupDestinationId($this->sourceIdValues));
$this->getEventDispatcher()->dispatch(MigrateEvents::POST_ROW_SAVE, new MigratePostRowSaveEvent($this->migration, $row, $destination_id_values));
$this->getEventDispatcher()->dispatch(MigrateEvents::POST_ROW_SAVE, new MigratePostRowSaveEvent($this->migration, $this->message, $row, $destination_id_values));
if ($destination_id_values) {
// We do not save an idMap entry for config.
if ($destination_id_values !== TRUE) {
@ -256,9 +262,6 @@ class MigrateExecutable implements MigrateExecutableInterface {
$this->handleException($e);
}
}
if ($high_water_property = $this->migration->getHighWaterProperty()) {
$this->migration->saveHighWater($row->getSourceProperty($high_water_property['name']));
}
// Reset row properties.
unset($sourceValues, $destinationValues);
@ -288,7 +291,7 @@ class MigrateExecutable implements MigrateExecutableInterface {
}
}
$this->getEventDispatcher()->dispatch(MigrateEvents::POST_IMPORT, new MigrateImportEvent($this->migration));
$this->getEventDispatcher()->dispatch(MigrateEvents::POST_IMPORT, new MigrateImportEvent($this->migration, $this->message));
$this->migration->setStatus(MigrationInterface::STATUS_IDLE);
return $return;
}
@ -348,10 +351,6 @@ class MigrateExecutable implements MigrateExecutableInterface {
break;
}
}
// If rollback completed successfully, reset the high water mark.
if ($return == MigrationInterface::RESULT_COMPLETED) {
$this->migration->saveHighWater(NULL);
}
// Notify modules that rollback attempt was complete.
$this->getEventDispatcher()->dispatch(MigrateEvents::POST_ROLLBACK, new MigrateRollbackEvent($this->migration));
@ -476,30 +475,44 @@ class MigrateExecutable implements MigrateExecutableInterface {
}
if ($pct_memory > $threshold) {
$this->message->display(
$this->t('Memory usage is @usage (@pct% of limit @limit), reclaiming memory.',
array('@pct' => round($pct_memory * 100),
'@usage' => $this->formatSize($usage),
'@limit' => $this->formatSize($this->memoryLimit))),
'warning');
$this->t(
'Memory usage is @usage (@pct% of limit @limit), reclaiming memory.',
array(
'@pct' => round($pct_memory * 100),
'@usage' => $this->formatSize($usage),
'@limit' => $this->formatSize($this->memoryLimit),
)
),
'warning'
);
$usage = $this->attemptMemoryReclaim();
$pct_memory = $usage / $this->memoryLimit;
// Use a lower threshold - we don't want to be in a situation where we keep
// coming back here and trimming a tiny amount
if ($pct_memory > (0.90 * $threshold)) {
$this->message->display(
$this->t('Memory usage is now @usage (@pct% of limit @limit), not enough reclaimed, starting new batch',
array('@pct' => round($pct_memory * 100),
'@usage' => $this->formatSize($usage),
'@limit' => $this->formatSize($this->memoryLimit))),
'warning');
$this->t(
'Memory usage is now @usage (@pct% of limit @limit), not enough reclaimed, starting new batch',
array(
'@pct' => round($pct_memory * 100),
'@usage' => $this->formatSize($usage),
'@limit' => $this->formatSize($this->memoryLimit),
)
),
'warning'
);
return TRUE;
}
else {
$this->message->display(
$this->t('Memory usage is now @usage (@pct% of limit @limit), reclaimed enough, continuing',
array('@pct' => round($pct_memory * 100),
'@usage' => $this->formatSize($usage),
'@limit' => $this->formatSize($this->memoryLimit))),
$this->t(
'Memory usage is now @usage (@pct% of limit @limit), reclaimed enough, continuing',
array(
'@pct' => round($pct_memory * 100),
'@usage' => $this->formatSize($usage),
'@limit' => $this->formatSize($this->memoryLimit),
)
),
'warning');
return FALSE;
}

View file

@ -1,187 +0,0 @@
<?php
namespace Drupal\migrate;
use Drupal\Component\Graph\Graph;
use Drupal\Component\Uuid\UuidInterface;
use Drupal\Core\Config\ConfigFactoryInterface;
use Drupal\Core\Config\Entity\ConfigEntityStorage;
use Drupal\Core\Entity\EntityTypeInterface;
use Drupal\Core\Entity\Query\QueryFactoryInterface;
use Drupal\Core\Language\LanguageManagerInterface;
use Symfony\Component\DependencyInjection\ContainerInterface;
/**
* Storage for migration entities.
*/
class MigrationStorage extends ConfigEntityStorage implements MigrateBuildDependencyInterface {
/**
* The entity query factory service.
*
* @var \Drupal\Core\Entity\Query\QueryFactoryInterface
*/
protected $queryFactory;
/**
* Constructs a MigrationStorage object.
*
* @param \Drupal\Core\Entity\EntityTypeInterface $entity_type
* An entity type definition.
* @param \Drupal\Core\Config\ConfigFactoryInterface $config_factory
* The config factory service.
* @param \Drupal\Component\Uuid\UuidInterface $uuid_service
* The UUID service.
* @param \Drupal\Core\Language\LanguageManagerInterface $language_manager
* The language manager.
* @param \Drupal\Core\Entity\Query\QueryFactoryInterface $query_factory
* The entity query factory service.
*/
public function __construct(EntityTypeInterface $entity_type, ConfigFactoryInterface $config_factory, UuidInterface $uuid_service, LanguageManagerInterface $language_manager, QueryFactoryInterface $query_factory) {
parent::__construct($entity_type, $config_factory, $uuid_service, $language_manager);
$this->queryFactory = $query_factory;
}
/**
* {@inheritdoc}
*/
public static function createInstance(ContainerInterface $container, EntityTypeInterface $entity_type) {
return new static(
$entity_type,
$container->get('config.factory'),
$container->get('uuid'),
$container->get('language_manager'),
$container->get('entity.query.config')
);
}
/**
* {@inheritdoc}
*/
public function loadMultiple(array $ids = NULL) {
if ($ids) {
$ids = $this->getVariantIds($ids);
}
/** @var \Drupal\migrate\Plugin\MigrationInterface[] $migrations */
$migrations = parent::loadMultiple($ids);
foreach ($migrations as $migration) {
$dependencies = array_map([$this, 'getVariantIds'], $migration->getMigrationDependencies());
$migration->set('migration_dependencies', $dependencies);
}
// Build an array of dependencies and set the order of the migrations.
return $this->buildDependencyMigration($migrations, []);
}
/**
* Splices variant IDs into a list of migration IDs.
*
* IDs which match the template_id:* pattern are shorthand for every variant
* of template_id. This method queries for those variant IDs and splices them
* into the original list.
*
* @param string[] $ids
* A set of migration IDs.
*
* @return string[]
* The expanded list of IDs.
*/
public function getVariantIds(array $ids) {
// Re-index the array numerically, since we need to limit the loop by size.
$ids = array_values($ids);
$index = 0;
while ($index < count($ids)) {
if (substr($ids[$index], -2) == ':*') {
$template_id = substr($ids[$index], 0, -2);
$variants = $this->queryFactory->get($this->entityType, 'OR')
->condition('id', $template_id)
->condition('template', $template_id)
->execute();
array_splice($ids, $index, 1, $variants);
$index += count($variants);
}
else {
$index++;
}
}
return $ids;
}
/**
* {@inheritdoc}
*/
public function buildDependencyMigration(array $migrations, array $dynamic_ids) {
// Migration dependencies defined in the migration storage can be
// optional or required. If an optional dependency does not run, the current
// migration is still OK to go. Both optional and required dependencies
// (if run at all) must run before the current migration.
$dependency_graph = array();
$requirement_graph = array();
$different = FALSE;
foreach ($migrations as $migration) {
/** @var \Drupal\migrate\Plugin\MigrationInterface $migration */
$id = $migration->id();
$requirements[$id] = array();
$dependency_graph[$id]['edges'] = array();
$migration_dependencies = $migration->getMigrationDependencies();
if (isset($migration_dependencies['required'])) {
foreach ($migration_dependencies['required'] as $dependency) {
if (!isset($dynamic_ids[$dependency])) {
$this->addDependency($requirement_graph, $id, $dependency, $dynamic_ids);
}
$this->addDependency($dependency_graph, $id, $dependency, $dynamic_ids);
}
}
if (isset($migration_dependencies['optional'])) {
foreach ($migration_dependencies['optional'] as $dependency) {
$different = TRUE;
$this->addDependency($dependency_graph, $id, $dependency, $dynamic_ids);
}
}
}
$graph_object = new Graph($dependency_graph);
$dependency_graph = $graph_object->searchAndSort();
if ($different) {
$graph_object = new Graph($requirement_graph);
$requirement_graph = $graph_object->searchAndSort();
}
else {
$requirement_graph = $dependency_graph;
}
$weights = array();
foreach ($migrations as $migration_id => $migration) {
// Populate a weights array to use with array_multisort later.
$weights[] = $dependency_graph[$migration_id]['weight'];
if (!empty($requirement_graph[$migration_id]['paths'])) {
$migration->set('requirements', $requirement_graph[$migration_id]['paths']);
}
}
array_multisort($weights, SORT_DESC, SORT_NUMERIC, $migrations);
return $migrations;
}
/**
* Add one or more dependencies to a graph.
*
* @param array $graph
* The graph so far, passed by reference.
* @param int $id
* The migration ID.
* @param string $dependency
* The dependency string.
* @param array $dynamic_ids
* The dynamic ID mapping.
*/
protected function addDependency(array &$graph, $id, $dependency, $dynamic_ids) {
$dependencies = isset($dynamic_ids[$dependency]) ? $dynamic_ids[$dependency] : array($dependency);
if (!isset($graph[$id]['edges'])) {
$graph[$id]['edges'] = array();
}
$graph[$id]['edges'] += array_combine($dependencies, $dependencies);
}
}

View file

@ -0,0 +1,138 @@
<?php
namespace Drupal\migrate\Plugin\Discovery;
use Doctrine\Common\Annotations\AnnotationRegistry;
use Doctrine\Common\Reflection\StaticReflectionParser as BaseStaticReflectionParser;
use Drupal\Component\Annotation\AnnotationInterface;
use Drupal\Component\Annotation\Reflection\MockFileFinder;
use Drupal\Component\ClassFinder\ClassFinder;
use Drupal\Core\Plugin\Discovery\AnnotatedClassDiscovery;
use Drupal\migrate\Annotation\MultipleProviderAnnotationInterface;
/**
* Determines providers based on a class's and its parent's namespaces.
*
* @internal
* This is a temporary solution to the fact that migration source plugins have
* more than one provider. This functionality will be moved to core in
* https://www.drupal.org/node/2786355.
*/
class AnnotatedClassDiscoveryAutomatedProviders extends AnnotatedClassDiscovery {
/**
* A utility object that can use active autoloaders to find files for classes.
*
* @var \Doctrine\Common\Reflection\ClassFinderInterface
*/
protected $finder;
/**
* Constructs an AnnotatedClassDiscoveryAutomatedProviders object.
*
* @param string $subdir
* Either the plugin's subdirectory, for example 'Plugin/views/filter', or
* empty string if plugins are located at the top level of the namespace.
* @param \Traversable $root_namespaces
* An object that implements \Traversable which contains the root paths
* keyed by the corresponding namespace to look for plugin implementations.
* If $subdir is not an empty string, it will be appended to each namespace.
* @param string $plugin_definition_annotation_name
* The name of the annotation that contains the plugin definition.
* Defaults to 'Drupal\Component\Annotation\Plugin'.
* @param string[] $annotation_namespaces
* Additional namespaces to scan for annotation definitions.
*/
public function __construct($subdir, \Traversable $root_namespaces, $plugin_definition_annotation_name = 'Drupal\Component\Annotation\Plugin', array $annotation_namespaces = []) {
parent::__construct($subdir, $root_namespaces, $plugin_definition_annotation_name, $annotation_namespaces);
$this->finder = new ClassFinder();
}
/**
* {@inheritdoc}
*/
protected function prepareAnnotationDefinition(AnnotationInterface $annotation, $class, BaseStaticReflectionParser $parser = NULL) {
if (!($annotation instanceof MultipleProviderAnnotationInterface)) {
throw new \LogicException('AnnotatedClassDiscoveryAutomatedProviders annotations must implement \Drupal\migrate\Annotation\MultipleProviderAnnotationInterface');
}
$annotation->setClass($class);
$providers = $annotation->getProviders();
// Loop through all the parent classes and add their providers (which we
// infer by parsing their namespaces) to the $providers array.
do {
$providers[] = $this->getProviderFromNamespace($parser->getNamespaceName());
} while ($parser = StaticReflectionParser::getParentParser($parser, $this->finder));
$providers = array_unique(array_filter($providers, function ($provider) {
return $provider && $provider !== 'component';
}));
$annotation->setProviders($providers);
}
/**
* {@inheritdoc}
*/
public function getDefinitions() {
$definitions = array();
$reader = $this->getAnnotationReader();
// Clear the annotation loaders of any previous annotation classes.
AnnotationRegistry::reset();
// Register the namespaces of classes that can be used for annotations.
AnnotationRegistry::registerLoader('class_exists');
// Search for classes within all PSR-0 namespace locations.
foreach ($this->getPluginNamespaces() as $namespace => $dirs) {
foreach ($dirs as $dir) {
if (file_exists($dir)) {
$iterator = new \RecursiveIteratorIterator(
new \RecursiveDirectoryIterator($dir, \RecursiveDirectoryIterator::SKIP_DOTS)
);
foreach ($iterator as $fileinfo) {
if ($fileinfo->getExtension() == 'php') {
if ($cached = $this->fileCache->get($fileinfo->getPathName())) {
if (isset($cached['id'])) {
// Explicitly unserialize this to create a new object instance.
$definitions[$cached['id']] = unserialize($cached['content']);
}
continue;
}
$sub_path = $iterator->getSubIterator()->getSubPath();
$sub_path = $sub_path ? str_replace(DIRECTORY_SEPARATOR, '\\', $sub_path) . '\\' : '';
$class = $namespace . '\\' . $sub_path . $fileinfo->getBasename('.php');
// The filename is already known, so there is no need to find the
// file. However, StaticReflectionParser needs a finder, so use a
// mock version.
$finder = MockFileFinder::create($fileinfo->getPathName());
$parser = new BaseStaticReflectionParser($class, $finder, FALSE);
/** @var $annotation \Drupal\Component\Annotation\AnnotationInterface */
if ($annotation = $reader->getClassAnnotation($parser->getReflectionClass(), $this->pluginDefinitionAnnotationName)) {
$this->prepareAnnotationDefinition($annotation, $class, $parser);
$id = $annotation->getId();
$content = $annotation->get();
$definitions[$id] = $content;
// Explicitly serialize this to create a new object instance.
$this->fileCache->set($fileinfo->getPathName(), ['id' => $id, 'content' => serialize($content)]);
}
else {
// Store a NULL object, so the file is not reparsed again.
$this->fileCache->set($fileinfo->getPathName(), [NULL]);
}
}
}
}
}
}
// Don't let annotation loaders pile up.
AnnotationRegistry::reset();
return $definitions;
}
}

View file

@ -0,0 +1,99 @@
<?php
namespace Drupal\migrate\Plugin\Discovery;
use Drupal\Component\Plugin\Discovery\DiscoveryInterface;
use Drupal\Component\Plugin\Discovery\DiscoveryTrait;
/**
* Remove plugin definitions with non-existing providers.
*
* @internal
* This is a temporary solution to the fact that migration source plugins have
* more than one provider. This functionality will be moved to core in
* https://www.drupal.org/node/2786355.
*/
class ProviderFilterDecorator implements DiscoveryInterface {
use DiscoveryTrait;
/**
* The Discovery object being decorated.
*
* @var \Drupal\Component\Plugin\Discovery\DiscoveryInterface
*/
protected $decorated;
/**
* A callable for testing if a provider exists.
*
* @var callable
*/
protected $providerExists;
/**
* Constructs a InheritProviderDecorator object.
*
* @param \Drupal\Component\Plugin\Discovery\DiscoveryInterface $decorated
* The object implementing DiscoveryInterface that is being decorated.
* @param callable $provider_exists
* A callable, gets passed a provider name, should return TRUE if the
* provider exists and FALSE if not.
*/
public function __construct(DiscoveryInterface $decorated, callable $provider_exists) {
$this->decorated = $decorated;
$this->providerExists = $provider_exists;
}
/**
* Removes plugin definitions with non-existing providers.
*
* @param mixed[] $definitions
* An array of plugin definitions (empty array if no definitions were
* found). Keys are plugin IDs.
* @param callable $provider_exists
* A callable, gets passed a provider name, should return TRUE if the
* provider exists and FALSE if not.
*
* @return array|\mixed[] $definitions
* An array of plugin definitions. If a definition is an array and has a
* provider key that provider is guaranteed to exist.
*/
public static function filterDefinitions(array $definitions, callable $provider_exists) {
// Besides what the caller accepts, we also accept core or component.
$provider_exists = function ($provider) use ($provider_exists) {
return in_array($provider, ['core', 'component']) || $provider_exists($provider);
};
return array_filter($definitions, function ($definition) use ($provider_exists) {
// Plugin definitions can be objects (for example, Typed Data) those will
// become empty array here and cause no problems.
$definition = (array) $definition + ['provider' => []];
// There can be one or many providers, handle them as multiple always.
$providers = (array) $definition['provider'];
return count($providers) == count(array_filter($providers, $provider_exists));
});
}
/**
* {@inheritdoc}
*/
public function getDefinitions() {
return static::filterDefinitions($this->decorated->getDefinitions(), $this->providerExists);
}
/**
* Passes through all unknown calls onto the decorated object.
*
* @param string $method
* The method to call on the decorated object.
* @param array $args
* Call arguments.
*
* @return mixed
* The return value from the method on the decorated object.
*/
public function __call($method, array $args) {
return call_user_func_array([$this->decorated, $method], $args);
}
}

View file

@ -0,0 +1,40 @@
<?php
namespace Drupal\migrate\Plugin\Discovery;
use Doctrine\Common\Reflection\StaticReflectionParser as BaseStaticReflectionParser;
/**
* Allows getting the reflection parser for the parent class.
*
* @internal
* This is a temporary solution to the fact that migration source plugins have
* more than one provider. This functionality will be moved to core in
* https://www.drupal.org/node/2786355.
*/
class StaticReflectionParser extends BaseStaticReflectionParser {
/**
* If the current class extends another, get the parser for the latter.
*
* @param \Doctrine\Common\Reflection\StaticReflectionParser $parser
* The current static parser.
* @param $finder
* The class finder. Must implement
* \Doctrine\Common\Reflection\ClassFinderInterface, but can do so
* implicitly (i.e., implements the interface's methods but not the actual
* interface).
*
* @return static|null
* The static parser for the parent if there's a parent class or NULL.
*/
public static function getParentParser(BaseStaticReflectionParser $parser, $finder) {
// Ensure the class has been parsed before accessing the parentClassName
// property.
$parser->parse();
if ($parser->parentClassName) {
return new static($parser->parentClassName, $finder, $parser->classAnnotationOptimize);
}
}
}

View file

@ -1,6 +1,7 @@
<?php
namespace Drupal\migrate\Plugin;
use Drupal\Component\Plugin\PluginInspectionInterface;
use Drupal\migrate\Row;

View file

@ -0,0 +1,80 @@
<?php
namespace Drupal\migrate\Plugin;
use Drupal\Core\Cache\CacheBackendInterface;
use Drupal\Core\Extension\ModuleHandlerInterface;
use Drupal\migrate\Plugin\Discovery\AnnotatedClassDiscoveryAutomatedProviders;
use Drupal\Core\Plugin\Discovery\ContainerDerivativeDiscoveryDecorator;
use Drupal\migrate\Plugin\Discovery\ProviderFilterDecorator;
/**
* Plugin manager for migrate source plugins.
*
* @see \Drupal\migrate\Plugin\MigrateSourceInterface
* @see \Drupal\migrate\Plugin\source\SourcePluginBase
* @see \Drupal\migrate\Annotation\MigrateSource
* @see plugin_api
*
* @ingroup migration
*/
class MigrateSourcePluginManager extends MigratePluginManager {
/**
* The class loader.
*
* @var object
*/
protected $classLoader;
/**
* MigrateSourcePluginManager constructor.
*
* @param string $type
* The type of the plugin: row, source, process, destination, entity_field,
* id_map.
* @param \Traversable $namespaces
* An object that implements \Traversable which contains the root paths
* keyed by the corresponding namespace to look for plugin implementations.
* @param \Drupal\Core\Cache\CacheBackendInterface $cache_backend
* Cache backend instance to use.
* @param \Drupal\Core\Extension\ModuleHandlerInterface $module_handler
* The module handler to invoke the alter hook with.
*/
public function __construct($type, \Traversable $namespaces, CacheBackendInterface $cache_backend, ModuleHandlerInterface $module_handler) {
parent::__construct($type, $namespaces, $cache_backend, $module_handler, 'Drupal\migrate\Annotation\MigrateSource');
}
/**
* {@inheritdoc}
*/
protected function getDiscovery() {
if (!$this->discovery) {
$discovery = new AnnotatedClassDiscoveryAutomatedProviders($this->subdir, $this->namespaces, $this->pluginDefinitionAnnotationName, $this->additionalAnnotationNamespaces);
$this->discovery = new ContainerDerivativeDiscoveryDecorator($discovery);
}
return $this->discovery;
}
/**
* Finds plugin definitions.
*
* @return array
* List of definitions to store in cache.
*
* @todo This is a temporary solution to the fact that migration source
* plugins have more than one provider. This functionality will be moved to
* core in https://www.drupal.org/node/2786355.
*/
protected function findDefinitions() {
$definitions = $this->getDiscovery()->getDefinitions();
foreach ($definitions as $plugin_id => &$definition) {
$this->processDefinition($definition, $plugin_id);
}
$this->alterDefinitions($definitions);
return ProviderFilterDecorator::filterDefinitions($definitions, function ($provider) {
return $this->providerExists($provider);
});
}
}

View file

@ -125,27 +125,6 @@ class Migration extends PluginBase implements MigrationInterface, RequirementsIn
*/
protected $destinationIds = [];
/**
* Information on the property used as the high watermark.
*
* Array of 'name' & (optional) db 'alias' properties used for high watermark.
*
* @var array
*/
protected $highWaterProperty;
/**
* Indicate whether the primary system of record for this migration is the
* source, or the destination (Drupal). In the source case, migration of
* an existing object will completely replace the Drupal object with data from
* the source side. In the destination case, the existing Drupal object will
* be loaded, then changes from the source applied; also, rollback will not be
* supported.
*
* @var string
*/
protected $systemOfRecord = self::SOURCE;
/**
* Specify value of source_row_status for current map row. Usually set by
* MigrateFieldHandler implementations.
@ -154,11 +133,6 @@ class Migration extends PluginBase implements MigrationInterface, RequirementsIn
*/
protected $sourceRowStatus = MigrateIdMapInterface::STATUS_IMPORTED;
/**
* @var \Drupal\Core\KeyValueStore\KeyValueStoreInterface
*/
protected $highWaterStorage;
/**
* Track time of last import if TRUE.
*
@ -173,6 +147,13 @@ class Migration extends PluginBase implements MigrationInterface, RequirementsIn
*/
protected $requirements = [];
/**
* An optional list of tags, used by the plugin manager for filtering.
*
* @var array
*/
protected $migration_tags = [];
/**
* These migrations, if run, must be executed before this migration.
*
@ -285,7 +266,7 @@ class Migration extends PluginBase implements MigrationInterface, RequirementsIn
$this->destinationPluginManager = $destination_plugin_manager;
$this->idMapPluginManager = $idmap_plugin_manager;
foreach ($plugin_definition as $key => $value) {
foreach (NestedArray::mergeDeep($plugin_definition, $configuration) as $key => $value) {
$this->$key = $value;
}
}
@ -436,33 +417,6 @@ class Migration extends PluginBase implements MigrationInterface, RequirementsIn
return $this->idMapPlugin;
}
/**
* Get the high water storage object.
*
* @return \Drupal\Core\KeyValueStore\KeyValueStoreInterface
* The storage object.
*/
protected function getHighWaterStorage() {
if (!isset($this->highWaterStorage)) {
$this->highWaterStorage = \Drupal::keyValue('migrate:high_water');
}
return $this->highWaterStorage;
}
/**
* {@inheritdoc}
*/
public function getHighWater() {
return $this->getHighWaterStorage()->get($this->id());
}
/**
* {@inheritdoc}
*/
public function saveHighWater($high_water) {
$this->getHighWaterStorage()->set($this->id(), $high_water);
}
/**
* {@inheritdoc}
*/
@ -628,21 +582,6 @@ class Migration extends PluginBase implements MigrationInterface, RequirementsIn
return $this;
}
/**
* {@inheritdoc}
*/
public function getSystemOfRecord() {
return $this->systemOfRecord;
}
/**
* {@inheritdoc}
*/
public function setSystemOfRecord($system_of_record) {
$this->systemOfRecord = $system_of_record;
return $this;
}
/**
* {@inheritdoc}
*/
@ -662,7 +601,30 @@ class Migration extends PluginBase implements MigrationInterface, RequirementsIn
* {@inheritdoc}
*/
public function getMigrationDependencies() {
return ($this->migration_dependencies ?: []) + ['required' => [], 'optional' => []];
$this->migration_dependencies = ($this->migration_dependencies ?: []) + ['required' => [], 'optional' => []];
$this->migration_dependencies['optional'] = array_unique(array_merge($this->migration_dependencies['optional'], $this->findMigrationDependencies($this->process)));
return $this->migration_dependencies;
}
/**
* Find migration dependencies from the migration and the iterator plugins.
*
* @param $process
* @return array
*/
protected function findMigrationDependencies($process) {
$return = [];
foreach ($this->getProcessNormalized($process) as $process_pipeline) {
foreach ($process_pipeline as $plugin_configuration) {
if ($plugin_configuration['plugin'] == 'migration') {
$return = array_merge($return, (array) $plugin_configuration['migration']);
}
if ($plugin_configuration['plugin'] == 'iterator') {
$return = array_merge($return, $this->findMigrationDependencies($plugin_configuration['process']));
}
}
}
return $return;
}
/**
@ -692,13 +654,6 @@ class Migration extends PluginBase implements MigrationInterface, RequirementsIn
return $this->source;
}
/**
* {@inheritdoc}
*/
public function getHighWaterProperty() {
return $this->highWaterProperty;
}
/**
* {@inheritdoc}
*/
@ -713,4 +668,11 @@ class Migration extends PluginBase implements MigrationInterface, RequirementsIn
return $this->destinationIds;
}
/**
* {@inheritdoc}
*/
public function getMigrationTags() {
return $this->migration_tags;
}
}

View file

@ -10,16 +10,6 @@ use Drupal\Component\Plugin\PluginInspectionInterface;
*/
interface MigrationInterface extends PluginInspectionInterface, DerivativeInspectionInterface {
/**
* A constant used for systemOfRecord.
*/
const SOURCE = 'source';
/**
* A constant used for systemOfRecord.
*/
const DESTINATION = 'destination';
/**
* The migration is currently not running.
*/
@ -152,26 +142,6 @@ interface MigrationInterface extends PluginInspectionInterface, DerivativeInspec
*/
public function getIdMap();
/**
* The current value of the high water mark.
*
* The high water mark defines a timestamp stating the time the import was last
* run. If the mark is set, only content with a higher timestamp will be
* imported.
*
* @return int
* A Unix timestamp representing the high water mark.
*/
public function getHighWater();
/**
* Save the new high water mark.
*
* @param int $high_water
* The high water timestamp.
*/
public function saveHighWater($high_water);
/**
* Check if all source rows from this migration have been processed.
*
@ -283,24 +253,6 @@ interface MigrationInterface extends PluginInspectionInterface, DerivativeInspec
*/
public function mergeProcessOfProperty($property, array $process_of_property);
/**
* Get the current system of record of the migration.
*
* @return string
* The current system of record of the migration.
*/
public function getSystemOfRecord();
/**
* Set the system of record for the migration.
*
* @param string $system_of_record
* The system of record of the migration.
*
* @return $this
*/
public function setSystemOfRecord($system_of_record);
/**
* Checks if the migration should track time of last import.
*
@ -343,18 +295,6 @@ interface MigrationInterface extends PluginInspectionInterface, DerivativeInspec
*/
public function getSourceConfiguration();
/**
* Get information on the property used as the high watermark.
*
* Array of 'name' & (optional) db 'alias' properties used for high watermark.
*
* @see Drupal\migrate\Plugin\migrate\source\SqlBase::initializeIterator()
*
* @return array
* The property used as the high watermark.
*/
public function getHighWaterProperty();
/**
* If true, track time of last import.
*
@ -374,4 +314,12 @@ interface MigrationInterface extends PluginInspectionInterface, DerivativeInspec
*/
public function getDestinationIds();
/**
* The migration tags.
*
* @return array
* Migration tags.
*/
public function getMigrationTags();
}

View file

@ -9,6 +9,7 @@ use Drupal\Core\Extension\ModuleHandlerInterface;
use Drupal\Core\Language\LanguageManagerInterface;
use Drupal\Core\Plugin\DefaultPluginManager;
use Drupal\Core\Plugin\Discovery\ContainerDerivativeDiscoveryDecorator;
use Drupal\migrate\Plugin\Discovery\ProviderFilterDecorator;
use Drupal\Core\Plugin\Discovery\YamlDirectoryDiscovery;
use Drupal\Core\Plugin\Factory\ContainerFactory;
use Drupal\migrate\MigrateBuildDependencyInterface;
@ -68,7 +69,15 @@ class MigrationPluginManager extends DefaultPluginManager implements MigrationPl
}, $this->moduleHandler->getModuleDirectories());
$yaml_discovery = new YamlDirectoryDiscovery($directories, 'migrate');
$this->discovery = new ContainerDerivativeDiscoveryDecorator($yaml_discovery);
// This gets rid of migrations which try to use a non-existent source
// plugin. The common case for this is if the source plugin has, or
// specifies, a non-existent provider.
$only_with_source_discovery = new NoSourcePluginDecorator($yaml_discovery);
// This gets rid of migrations with explicit providers set if one of the
// providers do not exist before we try to use a potentially non-existing
// deriver. This is a rare case.
$filtered_discovery = new ProviderFilterDecorator($only_with_source_discovery, [$this->moduleHandler, 'moduleExists']);
$this->discovery = new ContainerDerivativeDiscoveryDecorator($filtered_discovery);
}
return $this->discovery;
}
@ -77,7 +86,7 @@ class MigrationPluginManager extends DefaultPluginManager implements MigrationPl
* {@inheritdoc}
*/
public function createInstance($plugin_id, array $configuration = array()) {
$instances = $this->createInstances([$plugin_id], $configuration);
$instances = $this->createInstances([$plugin_id], [$plugin_id => $configuration]);
return reset($instances);
}
@ -228,4 +237,25 @@ class MigrationPluginManager extends DefaultPluginManager implements MigrationPl
return Migration::create(\Drupal::getContainer(), [], $id, $definition);
}
/**
* Finds plugin definitions.
*
* @return array
* List of definitions to store in cache.
*
* @todo This is a temporary solution to the fact that migration source
* plugins have more than one provider. This functionality will be moved to
* core in https://www.drupal.org/node/2786355.
*/
protected function findDefinitions() {
$definitions = $this->getDiscovery()->getDefinitions();
foreach ($definitions as $plugin_id => &$definition) {
$this->processDefinition($definition, $plugin_id);
}
$this->alterDefinitions($definitions);
return ProviderFilterDecorator::filterDefinitions($definitions, function ($provider) {
return $this->providerExists($provider);
});
}
}

View file

@ -0,0 +1,58 @@
<?php
namespace Drupal\migrate\Plugin;
use Drupal\Component\Plugin\Discovery\DiscoveryInterface;
use Drupal\Component\Plugin\Discovery\DiscoveryTrait;
/**
* Remove definitions which refer to a non-existing source plugin.
*/
class NoSourcePluginDecorator implements DiscoveryInterface {
use DiscoveryTrait;
/**
* The Discovery object being decorated.
*
* @var \Drupal\Component\Plugin\Discovery\DiscoveryInterface
*/
protected $decorated;
/**
* Constructs a NoSourcePluginDecorator object.
*
* @param \Drupal\Component\Plugin\Discovery\DiscoveryInterface $decorated
* The object implementing DiscoveryInterface that is being decorated.
*/
public function __construct(DiscoveryInterface $decorated) {
$this->decorated = $decorated;
}
/**
* {@inheritdoc}
*/
public function getDefinitions() {
/** @var \Drupal\Component\Plugin\PluginManagerInterface $source_plugin_manager */
$source_plugin_manager = \Drupal::service('plugin.manager.migrate.source');
return array_filter($this->decorated->getDefinitions(), function (array $definition) use ($source_plugin_manager) {
return $source_plugin_manager->hasDefinition($definition['source']['plugin']);
});
}
/**
* Passes through all unknown calls onto the decorated object.
*
* @param string $method
* The method to call on the decorated object.
* @param array $args
* Call arguments.
*
* @return mixed
* The return value from the method on the decorated object.
*/
public function __call($method, array $args) {
return call_user_func_array([$this->decorated, $method], $args);
}
}

View file

@ -0,0 +1,94 @@
<?php
namespace Drupal\migrate\Plugin;
use Drupal\migrate\Event\ImportAwareInterface;
use Drupal\migrate\Event\MigrateEvents;
use Drupal\migrate\Event\MigrateImportEvent;
use Drupal\migrate\Event\MigrateRollbackEvent;
use Drupal\migrate\Event\RollbackAwareInterface;
use Symfony\Component\EventDispatcher\EventSubscriberInterface;
/**
* Event subscriber to forward Migrate events to source and destination plugins.
*/
class PluginEventSubscriber implements EventSubscriberInterface {
/**
* Tries to invoke event handling methods on source and destination plugins.
*
* @param string $method
* The method to invoke.
* @param \Drupal\migrate\Event\MigrateImportEvent|\Drupal\migrate\Event\MigrateRollbackEvent $event
* The event that has triggered the invocation.
* @param string $plugin_interface
* The interface which plugins must implement in order to be invoked.
*/
protected function invoke($method, $event, $plugin_interface) {
$migration = $event->getMigration();
$source = $migration->getSourcePlugin();
if ($source instanceof $plugin_interface) {
call_user_func([$source, $method], $event);
}
$destination = $migration->getDestinationPlugin();
if ($destination instanceof $plugin_interface) {
call_user_func([$destination, $method], $event);
}
}
/**
* Forwards pre-import events to the source and destination plugins.
*
* @param \Drupal\migrate\Event\MigrateImportEvent $event
* The import event.
*/
public function preImport(MigrateImportEvent $event) {
$this->invoke('preImport', $event, ImportAwareInterface::class);
}
/**
* Forwards post-import events to the source and destination plugins.
*
* @param \Drupal\migrate\Event\MigrateImportEvent $event
* The import event.
*/
public function postImport(MigrateImportEvent $event) {
$this->invoke('postImport', $event, ImportAwareInterface::class);
}
/**
* Forwards pre-rollback events to the source and destination plugins.
*
* @param \Drupal\migrate\Event\MigrateRollbackEvent $event
* The rollback event.
*/
public function preRollback(MigrateRollbackEvent $event) {
$this->invoke('preRollback', $event, RollbackAwareInterface::class);
}
/**
* Forwards post-rollback events to the source and destination plugins.
*
* @param \Drupal\migrate\Event\MigrateRollbackEvent $event
* The rollback event.
*/
public function postRollback(MigrateRollbackEvent $event) {
$this->invoke('postRollback', $event, RollbackAwareInterface::class);
}
/**
* {@inheritdoc}
*/
public static function getSubscribedEvents() {
$events = [];
$events[MigrateEvents::PRE_IMPORT][] = ['preImport'];
$events[MigrateEvents::POST_IMPORT][] = ['postImport'];
$events[MigrateEvents::PRE_ROLLBACK][] = ['preRollback'];
$events[MigrateEvents::POST_ROLLBACK][] = ['postRollback'];
return $events;
}
}

View file

@ -147,7 +147,7 @@ class EntityContentBase extends Entity {
* @param \Drupal\migrate\Row $row
* The row object to update from.
*
* @return NULL|\Drupal\Core\Entity\EntityInterface
* @return \Drupal\Core\Entity\EntityInterface|null
* An updated entity, or NULL if it's the same as the one passed in.
*/
protected function updateEntity(EntityInterface $entity, Row $row) {

View file

@ -0,0 +1,225 @@
<?php
namespace Drupal\migrate\Plugin\migrate\process;
use Drupal\Core\File\FileSystemInterface;
use Drupal\Core\Plugin\ContainerFactoryPluginInterface;
use Drupal\Core\StreamWrapper\LocalStream;
use Drupal\Core\StreamWrapper\StreamWrapperManagerInterface;
use Drupal\migrate\MigrateException;
use Drupal\migrate\MigrateExecutableInterface;
use Drupal\migrate\ProcessPluginBase;
use Drupal\migrate\Row;
use Symfony\Component\DependencyInjection\ContainerInterface;
/**
* Copy a file from one place into another.
*
* @MigrateProcessPlugin(
* id = "file_copy"
* )
*/
class FileCopy extends ProcessPluginBase implements ContainerFactoryPluginInterface {
/**
* The stream wrapper manager service.
*
* @var \Drupal\Core\StreamWrapper\StreamWrapperManagerInterface
*/
protected $streamWrapperManager;
/**
* The file system service.
*
* @var \Drupal\Core\File\FileSystemInterface
*/
protected $fileSystem;
/**
* Constructs a file_copy process plugin.
*
* @param array $configuration
* The plugin configuration.
* @param string $plugin_id
* The plugin ID.
* @param mixed $plugin_definition
* The plugin definition.
* @param \Drupal\Core\StreamWrapper\StreamWrapperManagerInterface $stream_wrappers
* The stream wrapper manager service.
* @param \Drupal\Core\File\FileSystemInterface $file_system
* The file system service.
*/
public function __construct(array $configuration, $plugin_id, array $plugin_definition, StreamWrapperManagerInterface $stream_wrappers, FileSystemInterface $file_system) {
$configuration += array(
'move' => FALSE,
'rename' => FALSE,
'reuse' => FALSE,
);
parent::__construct($configuration, $plugin_id, $plugin_definition);
$this->streamWrapperManager = $stream_wrappers;
$this->fileSystem = $file_system;
}
/**
* {@inheritdoc}
*/
public static function create(ContainerInterface $container, array $configuration, $plugin_id, $plugin_definition) {
return new static(
$configuration,
$plugin_id,
$plugin_definition,
$container->get('stream_wrapper_manager'),
$container->get('file_system')
);
}
/**
* {@inheritdoc}
*/
public function transform($value, MigrateExecutableInterface $migrate_executable, Row $row, $destination_property) {
// If we're stubbing a file entity, return a URI of NULL so it will get
// stubbed by the general process.
if ($row->isStub()) {
return NULL;
}
list($source, $destination) = $value;
// Ensure the source file exists, if it's a local URI or path.
if ($this->isLocalUri($source) && !file_exists($source)) {
throw new MigrateException("File '$source' does not exist");
}
// If the start and end file is exactly the same, there is nothing to do.
if ($this->isLocationUnchanged($source, $destination)) {
return $destination;
}
$replace = $this->getOverwriteMode();
// We attempt the copy/move first to avoid calling file_prepare_directory()
// any more than absolutely necessary.
$final_destination = $this->writeFile($source, $destination, $replace);
if ($final_destination) {
return $final_destination;
}
// If writeFile didn't work, make sure there's a writable directory in
// place.
$dir = $this->getDirectory($destination);
if (!file_prepare_directory($dir, FILE_CREATE_DIRECTORY | FILE_MODIFY_PERMISSIONS)) {
throw new MigrateException("Could not create or write to directory '$dir'");
}
$final_destination = $this->writeFile($source, $destination, $replace);
if ($final_destination) {
return $final_destination;
}
throw new MigrateException("File $source could not be copied to $destination");
}
/**
* Tries to move or copy a file.
*
* @param string $source
* The source path or URI.
* @param string $destination
* The destination path or URI.
* @param int $replace
* (optional) FILE_EXISTS_REPLACE (default) or FILE_EXISTS_RENAME.
*
* @return string|bool
* File destination on success, FALSE on failure.
*/
protected function writeFile($source, $destination, $replace = FILE_EXISTS_REPLACE) {
if ($this->configuration['move']) {
return file_unmanaged_move($source, $destination, $replace);
}
// Check if there is a destination available for copying. If there isn't,
// it already exists at the destination and the replace flag tells us to not
// replace it. In that case, return the original destination.
if (!($final_destination = file_destination($destination, $replace))) {
return $destination;
}
// We can't use file_unmanaged_copy because it will break with remote Urls.
if (@copy($source, $final_destination)) {
return $final_destination;
}
return FALSE;
}
/**
* Determines how to handle file conflicts.
*
* @return int
* FILE_EXISTS_REPLACE (default), FILE_EXISTS_RENAME, or FILE_EXISTS_ERROR
* depending on the current configuration.
*/
protected function getOverwriteMode() {
if (!empty($this->configuration['rename'])) {
return FILE_EXISTS_RENAME;
}
if (!empty($this->configuration['reuse'])) {
return FILE_EXISTS_ERROR;
}
return FILE_EXISTS_REPLACE;
}
/**
* Returns the directory component of a URI or path.
*
* For URIs like public://foo.txt, the full physical path of public://
* will be returned, since a scheme by itself will trip up certain file
* API functions (such as file_prepare_directory()).
*
* @param string $uri
* The URI or path.
*
* @return string|false
* The directory component of the path or URI, or FALSE if it could not
* be determined.
*/
protected function getDirectory($uri) {
$dir = $this->fileSystem->dirname($uri);
if (substr($dir, -3) == '://') {
return $this->fileSystem->realpath($dir);
}
return $dir;
}
/**
* Determines if the source and destination URIs represent identical paths.
*
* If either URI is a remote stream, will return FALSE.
*
* @param string $source
* The source URI.
* @param string $destination
* The destination URI.
*
* @return bool
* TRUE if the source and destination URIs refer to the same physical path,
* otherwise FALSE.
*/
protected function isLocationUnchanged($source, $destination) {
if ($this->isLocalUri($source) && $this->isLocalUri($destination)) {
return $this->fileSystem->realpath($source) === $this->fileSystem->realpath($destination);
}
return FALSE;
}
/**
* Determines if the given URI or path is considered local.
*
* A URI or path is considered local if it either has no scheme component,
* or the scheme is implemented by a stream wrapper which extends
* \Drupal\Core\StreamWrapper\LocalStream.
*
* @param string $uri
* The URI or path to test.
*
* @return bool
*/
protected function isLocalUri($uri) {
$scheme = $this->fileSystem->uriScheme($uri);
return $scheme === FALSE || $this->streamWrapperManager->getViaScheme($scheme) instanceof LocalStream;
}
}

View file

@ -0,0 +1,60 @@
<?php
namespace Drupal\migrate\Plugin\migrate\process;
use Drupal\migrate\MigrateExecutableInterface;
use Drupal\migrate\MigrateException;
use Drupal\migrate\ProcessPluginBase;
use Drupal\migrate\Row;
use GuzzleHttp\Psr7\Uri;
/**
* Apply urlencoding to a URI.
*
* This is needed when the URI is to be opened by a later migration stage, and
* the source URI value is not already encoded.
*
* @MigrateProcessPlugin(
* id = "urlencode"
* )
*/
class UrlEncode extends ProcessPluginBase {
/**
* {@inheritdoc}
*/
public function transform($value, MigrateExecutableInterface $migrate_executable, Row $row, $destination_property) {
// Only apply to a full URL.
if (is_string($value) && strpos($value, '://') > 0) {
// URL encode everything after the hostname.
$parsed_url = parse_url($value);
// Fail on seriously malformed URLs.
if ($parsed_url === FALSE) {
throw new MigrateException("Value '$value' is not a valid URL");
}
// Iterate over specific pieces of the URL rawurlencoding each one.
$url_parts_to_encode = array('path', 'query', 'fragment');
foreach ($parsed_url as $parsed_url_key => $parsed_url_value) {
if (in_array($parsed_url_key, $url_parts_to_encode)) {
// urlencode() would convert spaces to + signs.
$urlencoded_parsed_url_value = rawurlencode($parsed_url_value);
// Restore special characters depending on which part of the URL this is.
switch ($parsed_url_key) {
case 'query':
$urlencoded_parsed_url_value = str_replace('%26', '&', $urlencoded_parsed_url_value);
break;
case 'path':
$urlencoded_parsed_url_value = str_replace('%2F', '/', $urlencoded_parsed_url_value);
break;
}
$parsed_url[$parsed_url_key] = $urlencoded_parsed_url_value;
}
}
$value = (string) Uri::fromParts($parsed_url);
}
return $value;
}
}

View file

@ -3,6 +3,8 @@
namespace Drupal\migrate\Plugin\migrate\source;
use Drupal\Core\Plugin\PluginBase;
use Drupal\migrate\Event\MigrateRollbackEvent;
use Drupal\migrate\Event\RollbackAwareInterface;
use Drupal\migrate\Plugin\MigrationInterface;
use Drupal\migrate\MigrateException;
use Drupal\migrate\MigrateSkipRowException;
@ -20,7 +22,7 @@ use Drupal\migrate\Row;
*
* @ingroup migration
*/
abstract class SourcePluginBase extends PluginBase implements MigrateSourceInterface {
abstract class SourcePluginBase extends PluginBase implements MigrateSourceInterface, RollbackAwareInterface {
/**
* The module handler service.
@ -36,15 +38,6 @@ abstract class SourcePluginBase extends PluginBase implements MigrateSourceInter
*/
protected $migration;
/**
* The name and type of the highwater property in the source.
*
* @var array
*
* @see $originalHighwater
*/
protected $highWaterProperty;
/**
* The current row from the query.
*
@ -59,10 +52,27 @@ abstract class SourcePluginBase extends PluginBase implements MigrateSourceInter
*/
protected $currentSourceIds;
/**
* Information on the property used as the high-water mark.
*
* Array of 'name' and (optional) db 'alias' properties used for high-water
* mark.
*
* @var array
*/
protected $highWaterProperty = [];
/**
* The key-value storage for the high-water value.
*
* @var \Drupal\Core\KeyValueStore\KeyValueStoreInterface
*/
protected $highWaterStorage;
/**
* The high water mark at the beginning of the import operation.
*
* If the source has a property for tracking changes (like Drupal ha
* If the source has a property for tracking changes (like Drupal has
* node.changed) then this is the highest value of those imported so far.
*
* @var int
@ -141,15 +151,18 @@ abstract class SourcePluginBase extends PluginBase implements MigrateSourceInter
$this->migration = $migration;
// Set up some defaults based on the source configuration.
$this->cacheCounts = !empty($configuration['cache_counts']);
$this->skipCount = !empty($configuration['skip_count']);
foreach (['cacheCounts' => 'cache_counts', 'skipCount' => 'skip_count', 'trackChanges' => 'track_changes'] as $property => $config_key) {
if (isset($configuration[$config_key])) {
$this->$property = (bool) $configuration[$config_key];
}
}
$this->cacheKey = !empty($configuration['cache_key']) ? $configuration['cache_key'] : NULL;
$this->trackChanges = !empty($configuration['track_changes']) ? $configuration['track_changes'] : FALSE;
$this->idMap = $this->migration->getIdMap();
$this->highWaterProperty = !empty($configuration['high_water_property']) ? $configuration['high_water_property'] : FALSE;
// Pull out the current highwater mark if we have a highwater property.
if ($this->highWaterProperty = $this->migration->getHighWaterProperty()) {
$this->originalHighWater = $this->migration->getHighWater();
if ($this->highWaterProperty) {
$this->originalHighWater = $this->getHighWater();
}
// Don't allow the use of both highwater and track changes together.
@ -324,6 +337,10 @@ abstract class SourcePluginBase extends PluginBase implements MigrateSourceInter
if (!$row->getIdMap() || $row->needsUpdate() || $this->aboveHighwater($row) || $this->rowChanged($row)) {
$this->currentRow = $row->freezeSource();
}
if ($this->getHighWaterProperty()) {
$this->saveHighWater($row->getSourceProperty($this->highWaterProperty['name']));
}
}
}
@ -337,7 +354,7 @@ abstract class SourcePluginBase extends PluginBase implements MigrateSourceInter
* TRUE if the highwater value in the row is greater than our current value.
*/
protected function aboveHighwater(Row $row) {
return $this->highWaterProperty && $row->getSourceProperty($this->highWaterProperty['name']) > $this->originalHighWater;
return $this->getHighWaterProperty() && $row->getSourceProperty($this->highWaterProperty['name']) > $this->originalHighWater;
}
/**
@ -384,7 +401,7 @@ abstract class SourcePluginBase extends PluginBase implements MigrateSourceInter
// If a refresh is requested, or we're not caching counts, ask the derived
// class to get the count from the source.
if ($refresh || !$this->cacheCounts) {
$count = $this->getIterator()->count();
$count = $this->doCount();
$this->getCache()->set($this->cacheKey, $count);
}
else {
@ -397,7 +414,7 @@ abstract class SourcePluginBase extends PluginBase implements MigrateSourceInter
else {
// No cached count, ask the derived class to count 'em up, and cache
// the result.
$count = $this->getIterator()->count();
$count = $this->doCount();
$this->getCache()->set($this->cacheKey, $count);
}
}
@ -417,4 +434,101 @@ abstract class SourcePluginBase extends PluginBase implements MigrateSourceInter
return $this->cache;
}
/**
* Gets the source count checking if the source is countable or using the
* iterator_count function.
*
* @return int
*/
protected function doCount() {
$iterator = $this->getIterator();
return $iterator instanceof \Countable ? $iterator->count() : iterator_count($this->initializeIterator());
}
/**
* Get the high water storage object.
*
* @return \Drupal\Core\KeyValueStore\KeyValueStoreInterface
* The storage object.
*/
protected function getHighWaterStorage() {
if (!isset($this->highWaterStorage)) {
$this->highWaterStorage = \Drupal::keyValue('migrate:high_water');
}
return $this->highWaterStorage;
}
/**
* The current value of the high water mark.
*
* The high water mark defines a timestamp stating the time the import was last
* run. If the mark is set, only content with a higher timestamp will be
* imported.
*
* @return int|null
* A Unix timestamp representing the high water mark, or NULL if no high
* water mark has been stored.
*/
protected function getHighWater() {
return $this->getHighWaterStorage()->get($this->migration->id());
}
/**
* Save the new high water mark.
*
* @param int $high_water
* The high water timestamp.
*/
protected function saveHighWater($high_water) {
$this->getHighWaterStorage()->set($this->migration->id(), $high_water);
}
/**
* Get information on the property used as the high watermark.
*
* Array of 'name' & (optional) db 'alias' properties used for high watermark.
*
* @see \Drupal\migrate\Plugin\migrate\source\SqlBase::initializeIterator()
*
* @return array
* The property used as the high watermark.
*/
protected function getHighWaterProperty() {
return $this->highWaterProperty;
}
/**
* Get the name of the field used as the high watermark.
*
* The name of the field qualified with an alias if available.
*
* @see \Drupal\migrate\Plugin\migrate\source\SqlBase::initializeIterator()
*
* @return string|null
* The name of the field for the high water mark, or NULL if not set.
*/
protected function getHighWaterField() {
if (!empty($this->highWaterProperty['name'])) {
return !empty($this->highWaterProperty['alias']) ?
$this->highWaterProperty['alias'] . '.' . $this->highWaterProperty['name'] :
$this->highWaterProperty['name'];
}
return NULL;
}
/**
* {@inheritdoc}
*/
public function preRollback(MigrateRollbackEvent $event) {
// Nothing to do in this implementation.
}
/**
* {@inheritdoc}
*/
public function postRollback(MigrateRollbackEvent $event) {
// Reset the high-water mark.
$this->saveHighWater(NULL);
}
}

View file

@ -161,7 +161,6 @@ abstract class SqlBase extends SourcePluginBase implements ContainerFactoryPlugi
*/
protected function initializeIterator() {
$this->prepareQuery();
$high_water_property = $this->migration->getHighWaterProperty();
// Get the key values, for potential use in joining to the map table.
$keys = array();
@ -213,15 +212,10 @@ abstract class SqlBase extends SourcePluginBase implements ContainerFactoryPlugi
}
// 2. If we are using high water marks, also include rows above the mark.
// But, include all rows if the high water mark is not set.
if (isset($high_water_property['name']) && ($high_water = $this->migration->getHighWater()) !== '') {
if (isset($high_water_property['alias'])) {
$high_water = $high_water_property['alias'] . '.' . $high_water_property['name'];
}
else {
$high_water = $high_water_property['name'];
}
$conditions->condition($high_water, $high_water, '>');
$condition_added = TRUE;
if ($this->getHighWaterProperty() && ($high_water = $this->getHighWater()) !== '') {
$high_water_field = $this->getHighWaterField();
$conditions->condition($high_water_field, $high_water, '>');
$this->query->orderBy($high_water_field);
}
if ($condition_added) {
$this->query->condition($conditions);