Source code for hybrid.core

# Copyright 2018 D-Wave Systems Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import operator
import logging
import threading
from collections import namedtuple, defaultdict
from copy import deepcopy

from plucky import merge
import dimod

from hybrid import traits
from hybrid.utils import min_sample, sample_as_dict, hstack_samplesets, vstack_samplesets, cpu_count
from hybrid.profiling import make_timeit, make_count
from hybrid.concurrency import Future, Present, Executor, immediate_executor, thread_executor

__all__ = [
    'SampleSet', 'State', 'States', 'Runnable', 'HybridSampler',
    'HybridRunnable', 'HybridProblemRunnable', 'HybridSubproblemRunnable',
    'stoppable'
]

logger = logging.getLogger(__name__)


class PliableDict(dict):
    """Dictionary subclass with attribute accessors acting as item accessors.

    Example:

        >>> d = PliableDict(x=1)
        >>> d.y = 2
        >>> d                     # doctest: +SKIP
        {'x': 1, 'y': 2}
        >>> d.z is None
        True
    """

    # some attribute lookups will be delegated to superclass, to handle things like pickling
    _delegated = frozenset(('__reduce_ex__', '__reduce__',
                            '__getstate__', '__setstate__',
                            '__getinitargs__', '__getnewargs__'))

    def __getattr__(self, name):
        if name in self._delegated:
            return super(PliableDict, self).__getattr__(name)

        return self.get(name)

    def __setattr__(self, name, value):
        self[name] = value


class SampleSet(dimod.SampleSet):
    """The `dimod.SampleSet` extended with a few helper methods.

    Note: this is basically a staging area for new `dimod.SampleSet` features
    before we merge them upstream.
    """

    def __init__(self, *args, **kwargs):
        if not args and not kwargs:
            # construct empty SampleSet
            empty = self.empty()
            super(SampleSet, self).__init__(empty.record, empty.variables,
                                            empty.info, empty.vartype)
        else:
            super(SampleSet, self).__init__(*args, **kwargs)

    @classmethod
    def empty(cls):
        return cls.from_samples([], vartype=dimod.SPIN, energy=0)

    def hstack(self, *others):
        """Combine the first sample in this SampleSet with first samples in all
        other SampleSets. Energy is reset to zero, and vartype is cast to the
        local vartype (first sampleset's vartype).
        """
        return hstack_samplesets(self, *others)

    def vstack(self, *others):
        return vstack_samplesets(self, *others)


[docs]class State(PliableDict): """Computation state passed along a branch between connected components. State is a :class:`dict` subclass and usually contains at least two keys: ``samples`` and ``problem``. Examples: >>> import dimod # Create a binary quadratic model >>> bqm = dimod.BinaryQuadraticModel({0: -1, 1: -1}, {(0, 1): 2}, 0.0, dimod.BINARY) >>> hybrid.core.State.from_sample(hybrid.utils.min_sample(bqm), bqm) # doctest: +SKIP {'problem': BinaryQuadraticModel({0: -1, 1: -1}, {(0, 1): 2}, 0.0, Vartype.BINARY), 'samples': SampleSet(rec.array([([0, 0], 0., 1)], dtype=[('sample', 'i1', (2,)), ('energy', '<f8'), ('num_occurrences', '<i4')]), [0, 1], {}, 'BINARY')} """ def __init__(self, *args, **kwargs): """State is a `PliableDict` (`dict` subclass) which usually contains at least two keys: `samples` and `problem`. """ super(State, self).__init__(*args, **kwargs) def copy(self): """Simple deep copy of itself. Functionally identical to `State.updated()`. """ return deepcopy(self)
[docs] def updated(self, **kwargs): """Return a (deep) copy of the state, updated from `kwargs`. This method has `dict.update` semantics with immutability of `sorted`. Currently an exception is the `debug` key, if it exists, for which a depth-unlimited recursive merge is executed. Example: >>> state = State() >>> state {} >>> newstate = state.updated(problem="test") >>> newstate {'problem': 'test'} """ overwrite = lambda a,b: b # use a special merge strategy for `debug` (op=overwrite, max_depth=None) debug = merge(self.get('debug', {}), kwargs.get('debug', {}), op=overwrite) if debug: self.setdefault('debug', PliableDict()) kwargs['debug'] = PliableDict(debug) return State(merge(self, kwargs, max_depth=1, op=overwrite))
def result(self): """Implement `concurrent.Future`-compatible result resolution interface. Also, explicitly defining this method prevents accidental definition of `State.result` method via attribute setters, which might prevent result resolution in some edge cases. """ return self
[docs] @classmethod def from_sample(cls, sample, bqm, **kwargs): """Convenience method for constructing a state from a raw (dict) sample. Energy is calculated from the binary quadratic model (BQM), and `State.problem` is also set to that BQM. Example: >>> import dimod >>> bqm = dimod.BQM.from_ising({}, {'ab': 0.5, 'bc': 0.5, 'ca': 0.5}) >>> state = State.from_sample({'a': -1, 'b': -1, 'c': -1}, bqm) """ return cls.from_samples([sample], bqm, **kwargs)
[docs] @classmethod def from_samples(cls, samples, bqm, **kwargs): """Convenience method for constructing a state from raw (dict) samples. Per-sample energy is calculated from the binary quadratic model (BQM), and `State.problem` is set to the BQM. Example: >>> import dimod >>> bqm = dimod.BQM.from_ising({}, {'ab': 0.5, 'bc': 0.5, 'ca': 0.5}) >>> state = State.from_samples([{'a': -1, 'b': -1, 'c': -1}, ... {'a': -1, 'b': -1, 'c': 1}], bqm) """ return cls(problem=bqm, samples=SampleSet.from_samples_bqm(samples, bqm), **kwargs)
@classmethod def from_subsample(cls, subsample, bqm, **kwargs): """Similar to :meth:`.from_sample`, but initializes `subproblem` and `subsamples`. """ return cls.from_subsamples([subsample], bqm, **kwargs) @classmethod def from_subsamples(cls, subsamples, bqm, **kwargs): """Similar to :meth:`.from_samples`, but initializes `subproblem` and `subsamples`. """ return cls(subproblem=bqm, subsamples=SampleSet.from_samples_bqm(subsamples, bqm), **kwargs) @classmethod def from_problem(cls, bqm, samples=None, **kwargs): """Convenience method for constructing a state from (possibly only) a BQM. """ if samples is None: samples = min_sample if callable(samples): samples_like = samples(bqm) else: samples_like = samples return cls.from_samples(samples_like, bqm, **kwargs) @classmethod def from_subproblem(cls, bqm, subsamples=None, **kwargs): """Convenience method for constructing a state from (possibly only) a subproblem BQM. """ if subsamples is None: subsamples = min_sample if callable(subsamples): subsamples_like = subsamples(bqm) else: subsamples_like = subsamples return cls.from_subsamples(subsamples_like, bqm, **kwargs)
[docs]class States(list): """List of states.""" def __init__(self, *args): super(States, self).__init__(args) def result(self): return self @property def first(self): return self[0] def updated(self, **kwargs): """Return a (deep) copy of the states, updated from `kwargs`.""" return States(*(state.updated(**kwargs) for state in self))
[docs]class Runnable(traits.StateTraits): """Components such as samplers and branches that can be run for an iteration. Args: **runopts (dict): Keyword arguments passed down to each `Runnable.run` call. Note: The base class :class:`~hybrid.core.Runnable` does not enforce traits validation. To enable validation, derive your subclass from one of the state structure, I/O dimensionality, or I/O validation mixins in :mod:`~hybrid.traits`. Examples: This example runs a tabu search on a binary quadratic model. An initial state is manually set to :math:`x=y=0, z=1; a=b=1, c=0` and an updated state is created by running the sampler for one iteration. >>> import dimod # Create a binary quadratic model >>> bqm = dimod.BinaryQuadraticModel({'x': 0.0, 'y': 0.0, 'z': 8.0, 'a': 2.0, 'b': 0.0, 'c': 6.0}, ... {('y', 'x'): 2.0, ('z', 'x'): -4.0, ('z', 'y'): -4.0, ... ('b', 'a'): 2.0, ('c', 'a'): -4.0, ('c', 'b'): -4.0, ('a', 'z'): -4.0}, ... -1.0, 'BINARY') >>> # Set up the sampler runnable >>> sampler = TabuProblemSampler(tenure=2, timeout=5) >>> # Run one iteration of the sampler >>> new_state = sampler.next(State.from_sample({'x': 0, 'y': 0, 'z': 1, 'a': 1, 'b': 1, 'c': 0}, bqm)) >>> print(new_state.samples) # doctest: +SKIP a b c x y z energy num_occ. 0 1 1 1 1 1 1 -1.0 1 [ 1 rows, 6 variables ] """ def __init__(self, **runopts): super(Runnable, self).__init__() self.runopts = runopts self.timers = defaultdict(list) self.timeit = make_timeit(self.timers, prefix=self.name, loglevel=logging.TRACE) self.counters = defaultdict(int) self.count = make_count(self.counters, prefix=self.name, loglevel=logging.TRACE) def __str__(self): return self.name def __repr__(self): return "{}()".format(self.name) def __iter__(self): return iter(tuple()) @property def name(self): """Return the :class:`Runnable` class name.""" return self.__class__.__name__
[docs] def init(self, state, **runopts): """Run prior to the first next/run, with the first state received. Default to NOP. """ pass
[docs] def next(self, state, **runopts): """Execute one blocking iteration of an instantiated :class:`Runnable` with a valid state as input. Args: state (:class:`State`): Computation state passed between connected components. Returns: :class:`State`: The new state. Examples: This code snippet runs one iteration of a sampler to produce a new state:: new_state = sampler.next(core.State.from_sample({'x': 0, 'y': 0}, bqm)) """ raise NotImplementedError
[docs] def error(self, exc): """Execute one blocking iteration of an instantiated :class:`Runnable` with an exception as input. Called when the previous component raised an exception instead of generating a new state. The default implementation raises again the input exception. Runnable errors must be explicitly silenced. """ raise exc
[docs] def halt(self): """Called by `stop()`. Override this method (instead of `stop`) to handle stopping of one blocking call of `next`. Defaults to NOP. """ pass
[docs] def dispatch(self, future, **kwargs): """Dispatch state from resolving `future` to either `next` or `error` methods. Args: state (:class:`concurrent.futures.Future`-like object): :class:`State` future. Returns state from :meth:`next` or :meth:`error`, or passes through an exception raised there. Blocks on state resolution and execution of :meth:`next` or :meth:`error`. """ with self.timeit('dispatch.resolve'): try: state = future.result() except Exception as exc: with self.timeit('dispatch.resolve.error'): return self.error(exc) if not getattr(self, '_initialized', False): with self.timeit('dispatch.init'): self.init(state, **kwargs) setattr(self, '_initialized', True) self.validate_input_state_traits(state) with self.timeit('dispatch.next'): new_state = self.next(state, **kwargs) self.validate_output_state_traits(new_state) return new_state
[docs] def run(self, state, **kwargs): """Execute the next step/iteration of an instantiated :class:`Runnable`. Accepts a state in a :class:`~concurrent.futures.Future`-like object and returns a new state in a :class:`~concurrent.futures.Future`-like object. Args: state (:class:`State`): Computation state future-like object passed between connected components. executor (:class:`~concurrent.futures.Executor`, optional, default=None): The Executor to which the execution of this block is scheduled. By default `hybrid.concurrency.thread_executor` is used. Examples: These two code snippets run one iteration of a sampler to produce a new state. The first is an asynchronous call and the second a blocking call. >>> sampler.run(State.from_sample(min_sample(bqm), bqm)) # doctest: +SKIP <Future at 0x20cbe22ea20 state=running> >>> sampler.run(State.from_sample(min_sample(bqm), bqm), ... executor=hybrid.immediate_executor) # doctest: +SKIP <Present at 0x20ca68cd2b0 state=finished returned State> """ # merge deferred local runopts with explicit kwarg options runopts = self.runopts.copy() runopts.update(kwargs) # based on merged runopts, decide on the executor executor = runopts.pop('executor', None) if executor is None: executor = thread_executor if not isinstance(executor, Executor): raise TypeError("expecting 'concurrent.futures.Executor' subclass " "instance for 'executor'") with self.timeit('dispatch'): return executor.submit(self.dispatch, state, **runopts)
[docs] def stop(self): """Terminate an iteration of an instantiated :class:`Runnable`.""" return self.halt()
def __or__(self, other): """Composition of runnable components (L-to-R) returns a new runnable Branch.""" return Branch(components=(self, other)) def __and__(self, other): """Parallel composition of runnable components returns new Branches.""" if isinstance(other, Branches): return Branches(self, *other) elif isinstance(other, Runnable): return Branches(self, other) else: raise TypeError("only Runnables can be composed into Branches")
def stoppable(cls): """Extends a `Runnable` subclass with a `stop_signal` semaphore/event, and amends the existing `halt` and `next` methods to signal stop via the semaphore, and reset the stop signal on run completion, respectively. Example: A Runnable block that accepts `timeout` and on `run` blocks for up to `timeout` seconds. It can be interrupted via call to `stop`, in which case blocks shorter than the timeout interval:: @stoppable class StoppableSleeper(Runnable): def next(self, state, timeout=None, **runopts): self.stop_signal.wait(timeout=timeout) return state >>> sleeper = StoppableSleeper(timeout=30) >>> sleeper.run(state) <Future at 0x7fbb575e6f60 state=running> >>> sleeper.stop() >>> sleeper.timers <snipped> 'dispatch.next': [13.224211193970405] """ if not issubclass(cls, Runnable): raise TypeError("Runnable subclass expected as the decorated class") orig_init = cls.__init__ orig_halt = cls.halt orig_next = cls.next def __init__(self, *args, **kwargs): orig_init(self, *args, **kwargs) self.stop_signal = threading.Event() def halt(self): result = orig_halt(self) self.stop_signal.set() return result def next(self, state, **runopts): result = orig_next(self, state, **runopts) self.stop_signal.clear() return result cls.__init__ = __init__ cls.halt = halt cls.next = next return cls
[docs]class HybridSampler(dimod.Sampler): """Produces a `dimod.Sampler` from a `hybrid.Runnable`-based sampler. Args: workflow (`Runnable`): Hybrid workflow, likely composed, that accepts a binary quadratic model in the input state and produces sample(s) in the output state. Example: This example produces a :std:doc:`dimod <oceandocs:docs_dimod/sdk_index>` sampler from :class:`~hybrid.samplers.TabuProblemSampler` and uses its `sample_ising` mixin to solve a simple Ising problem. >>> hybrid_sampler = TabuProblemSampler() >>> dimod_sampler = HybridSampler(hybrid_sampler) >>> solution = dimod_sampler.sample_ising({}, {'ab': 0.5, 'bc': 0.5, 'ca': 0.5}) >>> solution.first.energy -0.5 """ properties = None parameters = None def __init__(self, workflow): """Construct the sampler off of a (composed) `Runnable` BQM solver.""" if not isinstance(workflow, Runnable): raise TypeError("'sampler' should be 'hybrid.Runnable'") self._workflow = workflow self.parameters = {'initial_sample': []} self.properties = {} def sample(self, bqm, initial_sample=None, return_state=False, **params): """Sample from a binary quadratic model using composed runnable sampler. Args: bqm (:obj:`~dimod.BinaryQuadraticModel`): Binary quadratic model to be sampled from. initial_sample (dict, default=None): `bqm`-compatible sample used for initial state construction. Defaults to `hybrid.utils.min_sample(bqm)`. return_state (bool, optional, default=False): If True, the final state is added to :attr:`dimod.SampleSet.info` of the returned sample set. Note that if a `state` key already exists in the sample set then it is overwritten. **params (dict): Sampling parameters passed down to the underlying workflow as run-time parameters. Note: Sampling via hybrid workflow is run asynchronously, and a sample set is returned as soon as workflow starts running. A blocking result resolve occurres on the first attribute access to the returned sample set. Returns: :class:`~dimod.SampleSet`: Possibly yet unresolved sample set. """ if not isinstance(bqm, dimod.BinaryQuadraticModel): raise TypeError("'bqm' should be BinaryQuadraticModel") if initial_sample is None: initial_sample = min_sample(bqm) else: initial_sample = sample_as_dict(initial_sample) if len(initial_sample) != len(bqm): raise ValueError("size of 'initial_sample' incompatible with 'bqm'") initial_state = State.from_sample(initial_sample, bqm) final_state = self._workflow.run(initial_state, **params) def result_hook(state): resolved = state.result() ss = resolved.samples if return_state: # note: this creates a cyclic reference to `samples` from the # sample set via `info['state']`, but that shouldn't be a # problem for GC ss.info.update(state=resolved) return ss return dimod.SampleSet.from_future(final_state, result_hook=result_hook)
[docs]class HybridRunnable(Runnable): """Produces a `hybrid.Runnable` from a `dimod.Sampler` (dual of `HybridSampler`). The runnable samples from a problem defined in a state field named `fields[0]` and populates the state field referred to by `fields[1]`. Args: sampler (:class:`dimod.Sampler`): dimod-compatible sampler which is run on every iteration of the runnable. fields (tuple(str, str)): Input and output state field names. **sample_kwargs (dict): Sampler-specific parameters passed to sampler on every call. Example: This example creates a :class:`Runnable` from dimod sampler :std:doc:`TabuSampler <oceandocs:docs_tabu/sdk_index>`, runs it on an Ising model, and finds the lowest energy. >>> from tabu import TabuSampler >>> import dimod >>> bqm = dimod.BinaryQuadraticModel.from_ising({}, {'ab': 0.5, 'bc': 0.5, 'ca': 0.5}) >>> runnable = HybridRunnable(TabuSampler(), fields=('subproblem', 'subsamples'), timeout=100) >>> state0 = State(subproblem=bqm, subsamples=SampleSet.from_samples_bqm(min_sample(bqm), bqm)) >>> state = runnable.run(state0) >>> state.result()['subsamples'].first.energy # doctest: +SKIP -0.5 """ def __init__(self, sampler, fields, **sample_kwargs): super(HybridRunnable, self).__init__(**sample_kwargs) if not isinstance(sampler, dimod.Sampler): raise TypeError("'sampler' should be 'dimod.Sampler'") try: assert len(tuple(fields)) == 2 except: raise ValueError("'fields' should be two-tuple with input/output state fields") self.sampler = sampler self.input, self.output = fields def next(self, state, **sample_kwargs): response = self.sampler.sample(state[self.input], **sample_kwargs) return state.updated(**{self.output: response})
[docs]class HybridProblemRunnable(HybridRunnable): """Produces a `hybrid.Runnable` from a `dimod.Sampler` (dual of `HybridSampler`). The runnable that samples from `state.problem` and populates `state.samples`. See an example in :class:`hybrid.core.HybridRunnable`. An example of the duality with `HybridSampler` is:: HybridProblemRunnable(HybridSampler(TabuProblemSampler())) == TabuProblemSampler() """ def __init__(self, sampler, **sample_kwargs): super(HybridProblemRunnable, self).__init__( sampler, fields=('problem', 'samples'), **sample_kwargs)
[docs]class HybridSubproblemRunnable(HybridRunnable): """Produces a `hybrid.Runnable` from a `dimod.Sampler` (dual of `HybridSampler`). The runnable that samples from `state.subproblem` and populates `state.subsamples`. See an example in :class:`hybrid.core.HybridRunnable`. """ def __init__(self, sampler, **sample_kwargs): super(HybridSubproblemRunnable, self).__init__( sampler, fields=('subproblem', 'subsamples'), **sample_kwargs)
# deferred import, due to flow.* deps on core.* from hybrid.flow import Branch, Branches