darrc_template = """
--min-digits={settings.digits}
--slice {settings.slice_size_KiB:0.0f}K
# make crypto block size larger to reduce
# likelihood of duplicate ciphertext
--crypto-block 131072
# DO NOT specify the AES key here: this script is burned on every
# backup disc, in the clear
--key aes:
# don't back up caches, e.g. Firefox cache
--cache-directory-tagging
-v
create:
--compression=bzip2
-E "python3 {progname} {progargs} _create %p %b %n %e %c"
extract:
-O
-E "python3 {progname} {progargs} _extract %p %b %n %e %c"
list:
-E "python3 {progname} {progargs} _list %p %b %n %e %c"
test:
-E "python3 {progname} {progargs} _test %p %b %n %e %c"
isolate:
-E "python3 {progname} {progargs} _isolate %p %b %n %e %c"
"""

class Settings:
# vvvvvvvv    Below are variables for you to mess with    vvvvvvvvvvvvv

    burner_device = '/dev/null'

# SCRATCH_DIR must have (DATA_DISCS + PARITY_DISCS) * DISC_SIZE mebibytes free
# to run backup. SCRATCH_DIR must not exist when this script is run.
# SCRATCH_DIR must not be a subdirectory of the directory being backed up.
    scratch_dir = '/home/tmp/backup_scratch'

# This ballpark figure is used to calculate the number of digits to
# use when numbering archive slices.
    expected_data_size_GiB = 500.0

# Each redundancy set is composed of (DATA_DISCS + PARITY_DISCS) discs.
# These are like hard disk shelves with RAID, but with discs instead.
    data_discs = 3
    parity_discs = 2

# How many slices should be on each disc? I/O errors caused by media
# decay can truncate a slice; PAR1 then can't use the whole file. So
# let's be slightly wasteful.
    slices_per_disc = 500

# How much space is on a disc?
    # BluRay
    # https://superuser.com/a/565999 "256MB for defect management"
    disc_size_MiB = 23841 - 256
    # DVD
    ## disc_size_MiB = 4482
    # CD-R
    ## disc_size_MiB = 680

# ^^^^^^^^    Above are variables for you to mess with    ^^^^^^^^^^^

    # Reserved space is used for the filesystem, a copy of this program, a
    # README, and the par file that lists redundancy files.
    # Starting from http://stackoverflow.com/questions/468117, observed ISO +
    # Rock Ridge + Joliet filesystem overhead at 362K + 1.7K per filename.
    reserve_space_KiB = 10240

    # calculated settings

    @property
    def slices_per_set(self):
        return self.data_discs * self.slices_per_disc

    @property
    def total_set_count(self):
        return self.data_discs + self.parity_discs

    @property
    def scratch_free_needed_MiB(self):
        return self.total_set_count * self.disc_size_MiB

    def _calculate_digits(self):
        expected_frac_slices = (self.expected_data_size_GiB * 1024.0 *
                                self.slices_per_disc) / self.disc_size_MiB
        expected_slices = int(expected_frac_slices + 1)
        return len(str(expected_slices)) + 1
    def _set_digits(self, new_value):
        self._digits = new_value
    def _get_digits(self):
        if not hasattr(self, '_digits'):
            self._digits = self._calculate_digits()
        return self._digits
    # make digits settable so the unit tests can just set it without
    # worrying about what the right value of expected_data_size is for
    # their fantastic scenarios
    digits = property(_get_digits, _set_digits)

    @property
    def number_format(self):
        return '{:0' + str(self.digits) + '}'

    @property
    def _slice_size_not_counting_par_overhead_KiB(self):
        return ((self.disc_size_KiB - self.reserve_space_KiB) //
                self.slices_per_disc)

    @property
    def slice_size_KiB(self):
        # This allows for 32-character dar slice filenames.
        par_header_bytes = 96 + 120 * self.data_discs
        # Each pXX file has a par header; and for each data_discs
        # slices, there's a par file
        par_overhead_bytes = par_header_bytes * (1 + 1 / self.data_discs)
        par_overhead_KiB = (par_overhead_bytes + 1024) // 1024
        return self._slice_size_not_counting_par_overhead_KiB - par_overhead_KiB

    # auto-convert between _KiB and _MiB
    def __getattr__(self, name):
        if name in dir(self):
            return getattr(super(), name)
        else:
            if name.endswith('_MiB'):
                as_KiB = name.replace('_MiB', '_KiB')
                if as_KiB in dir(self):
                    return getattr(self, as_KiB) / 1024
            elif name.endswith('_KiB'):
                as_MiB = name.replace('_KiB', '_MiB')
                if as_MiB in dir(self):
                    return getattr(self, as_MiB) * 1024
            raise AttributeError(name)

    # -n switch turns this off
    actually_burn = True

from itertools import chain
import sys
import os
import shutil
import glob
import getopt
import subprocess
import tempfile
import contextlib
import unittest
import logging
import io
import re
import itertools
import random
import math
import pickle
import base64
try:
    from unittest.mock import Mock, patch, sentinel, call
except ImportError:
    from mock import Mock, patch, sentinel, call

def usage(settings):
    print("""
This script makes compressed, encrypted backups with {s.slice_size_MiB:0.2f} MiB \
slices striped
across sets of {s.total_set_count} {s.disc_size_MiB} MiB optical discs, \
each set containing {s.data_discs} data disc(s)
and {s.parity_discs} parity disc(s). It \
requires the following software (or later versions):
Python 3.2; mock 1.0 (included in Python 3.3); dar 2.5.4*; parchive 1.1;
growisofs 7.1; genisoimage 1.1.11.

* If you are encrypting, you need change 8e64f413. If you have dar
2.5.4 or later, you have change 8e64f413. If you don't (2.5.4 is not
yet released as of March 2016), you will need to build the
branch_2.5.x branch of dar yourself. See
<https://sourceforge.net/p/dar/bugs/183/> and
<https://sourceforge.net/p/dar/code/ci/8e64f413deb046064156792078060d2ee6ea4e5c/>.
If you are not using encryption, any recent dar will do (2.4.8 did
fine without encryption, for example).

When backing up, the directory {s.scratch_dir!r} should have
{s.scratch_free_needed_MiB} MiB of space free. \
When restoring, copy this script off of the optical
disc first; you'll need to switch optical discs during the backup.

If you don't like any of these settings, change this script. The
settings are toward the top.

Usage: python3 {progname} [-v] [-n] dar <dar parameters>

Dar parameters of note:
    Creating archive:   -c <archive basename> -R <dir with files to backup>
    Extracting archive: -x <archive basename>

You get 23 characters for the archive basename. (Sorry, that's ISO
9660.) See dar(1) about parameters you can give to dar. Don't get
fancy: only use the ones that tell dar which mode to operate in, and
which files to archive.  Otherwise this script will not form a
complete record of how dar was run.

The -v switch, before dar, means to be verbose and show the dar command
being executed and the darrc used. The -n switch, before dar, means don't
burn any discs: just make directories containing the files that would have
been burned. (This can use much more scratch space.)

""".format(s=settings, progname=sys.argv[0]),
        file=sys.stderr)

@contextlib.contextmanager
def working_directory(newcwd):
    oldcwd = os.getcwd()
    try:
        os.chdir(newcwd)
        yield
    finally:
        os.chdir(oldcwd)

class NotEnoughScratchSpace(Exception):
    pass

class ScratchAlreadyExists(Exception):
    pass

parity_volume_re = re.compile(r'.*\.[pqr][0-9][0-9]')

# This is a class not because it needs state, but because I didn't want to pass
# settings around all the time
class Darbrrb:
    def __init__(self, settings, progname, progopts=()):
        self.settings = settings
        self.progname = progname
        self.progopts = progopts
        self.log = logging.getLogger('darbrrb')

    def _run(self, *args):
        try_again = True
        while try_again:
            self.log.info('running command {!r}'.format(args))
            try:
                subprocess.check_call(args)
                try_again = False
            except subprocess.CalledProcessError as e:
                self.log.exception('an error was encountered '
                                   'when running command {!r}'.format(args))
                valid_input = False
                while not valid_input:
                    the_input = input('Something went wrong '
                                      'when running command {!r}. '
                                      'Try again? [Y/n] '.format(args))
                    if the_input == '':
                        valid_input = True
                        try_again = True
                    elif (the_input.startswith('y') or
                          the_input.startswith('Y')):
                        valid_input = True
                        try_again = True
                    elif (the_input.startswith('n') or
                          the_input.startswith('N')):
                        valid_input = True
                        try_again = False
                        self.log.error('re-raising the error')
                        raise
                    if not valid_input:
                        print('Did not understand your input. Asking again.')

    # for mockability
    def _copy(self, source, destination):
        shutil.copyfile(source, destination)

    @property
    def darrc_contents(self):
        progargs = []
        for o, v in self.progopts:
            if v:
                progargs.extend(o, v)
            else:
                progargs.append(o)
        return darrc_template.format(settings=self.settings,
                progname=os.path.join(self.settings.scratch_dir,
                        os.path.basename(self.progname)),
                progargs=' '.join(progargs))

    def readme(self, basename):
        if 'DARBRRB_ORIGINAL_ARGV' in os.environ:
            try:
                original_argv = pickle.loads(
                    base64.b64decode(os.environ['DARBRRB_ORIGINAL_ARGV']))
            except:
                original_argv = ['there was an error trying to find out']
        else:
            original_argv = ['not known']
        return """

This disc is part of a backup made by darbrrb, a tool that wraps the dar disk
archiver and the parchive file verification and repair tool to produce backups
with redundancy, for greater resilience against data loss due to backup media
failures or losses.

The {progname} script you find on this disc is a copy of the one used to make
the backup; some salient settings are written toward the top of this file.

darbrrb was run with these arguments:

{argv!r}

darbrrb ran dar with this darrc:

# ----------------
{contents}
# ----------------

The backup is split into redundancy sets of {s.total_set_count} discs. Out of
each set, dar archive slices are striped across the {s.data_discs} data
disc(s); par files with parity data for the dar slices are striped across the
{s.parity_discs} disc(s). Each disc can store {s.disc_size_MiB} MiB of data, and
each slice is {s.slice_size_MiB:0.2f} MiB in size.

To restore some files: first, make a directory somewhere with at least
{s.scratch_free_needed_MiB:0.0f} MiB free. Copy this script from a disc of the backup
into your directory. Run it with arguments like those above, but replace the
-c with a -x (you are extracting an archive instead of creating it), and
replace the value of the -R switch, which was the directory where all the
files were backed up from, with the directory to which you want the files
restored.

You'll be asked for discs from the backup.

About the files that may be on this disc:

* README.txt: this file.
* {progname}: the darbrrb script used to make the backup.
* {basename}.{one}.dar (e.g.): A dar slice file. This contains the data that
  was backed up.
* {basename}.{one}-{fddn}.par (e.g.): a par index file for a parity volume
  set made over several dar slice files. It contains checksums for each of
  the slice files in the set.
* {basename}.{one}-{fddn}.p01 (e.g.): a par parity volume file for the
  aforementioned parity volume set. This contains redundant data, such that
  if any of the dar slices in the set is missing or corrupted, it can be re-
  constructed.

""".format(argv=original_argv, s=self.settings,
           contents=self.darrc_contents,
           progname=os.path.basename(self.progname),
           basename=basename,
           one=self.settings.number_format.format(1),
           fddn=self.settings.number_format.format(self.settings.data_discs))
    # FIXME

    def dar(self, *args):
        # Perhaps darrc files can be non-ascii, but we haven't got any
        # non-ascii arguments to give here, so we'll stay on the safe side.
        indented_contents = self.darrc_contents.replace('\n', '\n        ')
        with open(os.path.join(self.settings.scratch_dir, 'darrc'),
                  'w', encoding='ascii') as darrc_file:
            self.log.info("""Contents of {name} follow:
{indented}
""".format(name=darrc_file.name, indented=indented_contents))
            darrc_file.write(self.darrc_contents)
            darrc_file.flush()
            # causes of this working_directory:
            # 1. when dar makes files, it will make them in the scratch_dir
            # 2. when dar calls this script, the _create and other methods
            #    below will have scratch_dir as their cwd.
            with working_directory(self.settings.scratch_dir):
                self._run('dar', *(args + ('-B', darrc_file.name)))

    def wait_for_empty_disc(self):
        # There are a hundred cooler ways to do this; in 2013, I don't know of
        # one that works on many distros and OSes, much less ten years from
        # now. But you'll probably still be able to press enter, some way.
        if self.settings.actually_burn:
            input("press enter when you have inserted an empty disc:")

    def written_disc_directory(self, disc_title):
        # Same as above. Now it's 2016, and all the ways I knew in
        # 2013 don't work any more, there are new ways. But you could
        # say "insert disc 2" in 1986, and you can say it today. And
        # get off my lawn!
        if self.settings.actually_burn:
            dir = input("insert and mount disc entitled {} and type the "
                        "directory where its files can be found: ".format(
                            disc_title))
            return dir
        else:
            return os.path.join(self.settings.scratch_dir, disc_title)

    # zb: zero-based; ob: one-based

    def last_set_directory(self, basename, disc_number_in_set_zb):
        if self.settings.actually_burn:
            dir = input("insert and mount disc {} from the last set of "
                        "backup {!r} and type the directory where its "
                        "files can be found: ".format(disc_number_in_set_zb + 1,
                                                      basename))
            return dir
        else:
            dirs = glob.glob(os.path.join(self.settings.scratch_dir,
                                          basename + '-*'))
            last_disc_dir = sorted(dirs)[-1]
            disc_in_last_set_dir = '{}{:03d}'.format(last_disc_dir[:-3],
                                                     disc_number_in_set_zb + 1)
            return disc_in_last_set_dir

    def disc_dir(self, disc):
        return '__disc{:04d}'.format(disc)

    def disc_dirs(self):
        return sorted(glob.glob('__disc*'))

    def disc_title(self, basename, set_number_zb, disc_in_set_number_zb):
        # Max ISO 9660 vol id length is 32. Leave room for numbers and 2 dashes.
        # +1: These numbers are 0-based, but we want the ones in the title 1-based.
        # If you change the format here, change code above in last_set_directory!
        return "{}-{:04d}-{:03d}".format(basename[:(32-4-3-2)],
                                         set_number_zb + 1,
                                         disc_in_set_number_zb + 1)

    def disc_title_for_slice(self, basename, dar_slice_number):
        set_number = math.floor((dar_slice_number - 1) /
                self.settings.slices_per_set)
        disc_in_set_number = ((dar_slice_number - 1) %
                              self.settings.data_discs)
        return self.disc_title(basename, set_number, disc_in_set_number)

    def disc_title_for_slice_and_disc(self, basename, dar_slice_number_ob, disc_in_set_number_zb):
        set_number_zb = math.floor((dar_slice_number_ob - 1) /
                self.settings.slices_per_set)
        return self.disc_title(basename, set_number_zb, disc_in_set_number_zb)

    def scratch_free_MiB(self):
        s = os.statvfs(self.settings.scratch_dir)
        return s.f_bavail * s.f_frsize // 1048576

    def ensure_free_space(self):
        free_space_MiB = self.scratch_free_MiB()
        if free_space_MiB < self.settings.scratch_free_needed_MiB:
            raise NotEnoughScratchSpace(self.settings.scratch_dir,
                                        self.settings.scratch_free_needed_MiB,
                                        free_space_MiB)

    def ensure_scratch(self):
        if os.path.exists(self.settings.scratch_dir):
            if not os.path.isdir(self.settings.scratch_dir):
                raise ScratchAlreadyExists()
        else:
            os.mkdir(self.settings.scratch_dir)
        for disc in range(1, self.settings.total_set_count + 1):
            os.mkdir(os.path.join(self.settings.scratch_dir,
                    self.disc_dir(disc)))
        self.ensure_free_space()
        # this is the copy of this program that dar will run
        self._copy(self.progname,
                   os.path.join(self.settings.scratch_dir,
                                os.path.basename(self.progname)))

    def _par_filename(self, basename, min_number, max_number):
        parformat = "{{}}.{0}-{0}.par".format(self.settings.number_format)
        return parformat.format(basename, min_number, max_number)

    def make_redundancy_files(self, basename, dar_files, max_number):
        nslices = len(dar_files)
        if nslices == 0:
            raise ValueError('no dar slices for parchive to operate on')
        min_number = max_number - nslices + 1
        parfilename = self._par_filename(basename, min_number, max_number)
        self._run(*(['parchive',
                     '-n{}'.format(self.settings.parity_discs),
                     'a', parfilename,
                ] + dar_files))
        return parfilename

    def burn(self, basename, slice_number, disc_in_set_number, dir, happening):
        if self.settings.actually_burn:
            self._run('growisofs', '-Z', self.settings.burner_device,
                      '-R', '-J', '-V',
                      self.disc_title_for_slice_and_disc(basename, slice_number,
                                                         disc_in_set_number),
                      dir)
        else:
            destination = os.path.join(self.settings.scratch_dir,
                    self.disc_title_for_slice_and_disc(basename, slice_number,
                                                       disc_in_set_number))
            self.log.info('not actually burning: moving files from {} to ' \
                    '{}'.format(dir, destination))
            os.mkdir(destination)
            for f in glob.glob(os.path.join(dir, '*')):
                shutil.move(f, os.path.join(destination, os.path.basename(f)))

    def _create(self, dir, basename, number, extension, happening):
        number = int(number)
        # note: dar has caused this function to be called; dar's cwd is
        # SCRATCH_DIR, hence so is ours
        dar_files_here = sorted(glob.glob('*.dar'))
        if len(dar_files_here) >= self.settings.data_discs or \
                happening == 'last_slice':
            parfilename = self.make_redundancy_files(
                    basename, dar_files_here, number)
            par_volumes = [f for f in os.listdir()
                           if parity_volume_re.match(f)]
            for d in self.disc_dirs():
                self._copy(parfilename, os.path.join(d, parfilename))
                with io.open(os.path.join(d, 'README.txt'), 'wt') as readme:
                    readme.write(self.readme(basename))
                this_program = os.path.basename(self.progname)
                self._copy(this_program, os.path.join(d, this_program))
            data_dirs = itertools.cycle(self.disc_dir(i+1)
                    for i in range(self.settings.data_discs))
            redundancy_dirs = itertools.cycle(self.disc_dir(i+1)
                    for i in range(self.settings.data_discs,
                            self.settings.total_set_count))
            for f, d in itertools.chain(
                    zip(dar_files_here, data_dirs),
                    zip(par_volumes, redundancy_dirs)):
                shutil.move(f, d)
        dars_on_discs = len(glob.glob(
                os.path.join(self.disc_dir(1), '*.dar')))
        size_if_we_dont_burn_KiB = (dars_on_discs + 1) * \
                self.settings.slice_size_KiB + \
                self.settings.reserve_space_KiB
        if size_if_we_dont_burn_KiB > self.settings.disc_size_KiB or \
                happening == 'last_slice':
            for i, d in enumerate(self.disc_dirs()):
                self.log.info("burning from {}".format(d))
                self.wait_for_empty_disc()
                self.burn(basename, number, i, d, happening)
                for fn in glob.glob(os.path.join(d, '*')):
                    os.unlink(fn)

    def _slice_name(self, basename, number, extension):
        return '{{}}.{{:0{}d}}.{{}}'.format(self.settings.digits).format(
            basename, number, extension)

    def _number_from_slice_name_ob(self, filename):
        return int(filename.split('.')[-2], 10)

    def _number_from_slice_name_zb(self, filename):
        return self._number_from_slice_name_ob(filename) - 1

    def _numbers_from_par_filename_ob(self, filename):
        # maybe.dots.here.XXXXX-YYYYY.par
        numbers = filename.split('.')[-2]
        first_s, last_s = numbers.split('-')
        first_ob = int(first_s, 10)
        last_ob = int(last_s, 10)
        return (first_ob, last_ob)

    def _numbers_from_par_filename_zb(self, filename):
        a, b = self._numbers_from_par_filename_ob(filename)
        return (a-1, b-1)

    def _last_parity_set_slices_zb(self, basename):
        # SIDE EFFECT: compels the insertion of the first disc in the
        # last set.
        #
        # Any disc in a set has all the pars from the set. The name of
        # the par file contains the slice numbers in the set. So WLOG
        # we ask for the first disc.
        disc_dir = self.last_set_directory(basename, 0)
        self.log.debug('first disc in last set is %r', disc_dir)
        last_par = sorted([x for x in os.listdir(disc_dir)
                           if x.endswith('.par')])[-1]
        self.log.debug('last_par is %r', last_par)
        return self._numbers_from_par_filename_zb(last_par)

    def _fetch_some_slices(self, basename, first_slice_zb, last_slice_zb=None):
        self.log.debug('_fetch_some_slices(%r, %r)', first_slice_zb, last_slice_zb)
        # we need entire parity sets, so if first_slice_zb is in the
        # middle of a set, we start at the beginning of the set
        first_slice_zb -= first_slice_zb % self.settings.data_discs
        set_number_zb = math.floor(first_slice_zb /
                                   self.settings.slices_per_set)
        self.log.debug('for slice %r (zb) et seq we want set (zb) %d',
                       first_slice_zb, set_number_zb)
        # MAYBE FIXME: we take the set of .par files on the first disc
        # of the set as authoritative; if any are missing I'm not sure
        # what would happen.
        disc_zb = 0
        disc_title = self.disc_title(basename, set_number_zb, disc_zb)
        disc_dir = self.written_disc_directory(disc_title)
        pars = sorted([x for x in os.listdir(disc_dir) if x.endswith('.par')])
        self.log.debug('pars: %r', pars)
        parity_set_ranges = [self._numbers_from_par_filename_zb(p) for p in pars]
        parity_sets_hereafter = [(a,b) for a,b in parity_set_ranges
                                 if a >= first_slice_zb]
        pars_hereafter = [self._par_filename(basename, a+1, b+1)
                          for a,b in parity_sets_hereafter]
        self.log.debug('pars_hereafter (after slice %d, %d in set %d): %r',
                       first_slice_zb,
                       first_slice_zb % self.settings.slices_per_set,
                       set_number_zb,
                       pars_hereafter)
        if last_slice_zb is None:
            max_last_slice_zb = parity_sets_hereafter[-1][-1]
            if (max_last_slice_zb - first_slice_zb) < (
                    0.5 * self.settings.slices_per_set):
                # no use being smart
                self.log.debug('less than half a set left, fetching the rest')
                last_slice_zb = max_last_slice_zb
            else:
                try:
                    with open(os.path.join(self.settings.scratch_dir,
                                           'last_fetched_fraction.txt')) as f:
                        last_fetched_set_zb = int(f.readline())
                        last_fetched_fraction = float(f.readline())
                except FileNotFoundError:
                    last_fetched_set_zb = set_number_zb
                    last_fetched_fraction = 0.03
                if last_fetched_set_zb != set_number_zb:
                    # this file is out of date
                    last_fetched_set_zb = set_number_zb
                    last_fetched_fraction = 0.03
                to_fetch_fraction = last_fetched_fraction * 3
                self.log.debug('this time we are fetching %6.3f of a set',
                               to_fetch_fraction)
                max_to_fetch = int((to_fetch_fraction *
                                    self.settings.slices_per_set) + 1)
                last_slice_zb = min(max_last_slice_zb,
                                    first_slice_zb + max_to_fetch)
                with open(os.path.join(self.settings.scratch_dir,
                                       'last_fetched_fraction.txt'), 'wt') as f:
                    print(last_fetched_set_zb, file=f)
                    print(to_fetch_fraction, file=f)
        # we need entire parity sets, so if last_slice_zb is in the
        # middle of a set, we end at the end of the set
        for a, b in parity_sets_hereafter:
            if last_slice_zb >= a and last_slice_zb <= b:
                last_slice_zb = b
                break
        else:
            self.log.error('could not find which parity set slice %d is in',
                           last_slice_zb)
        self.log.debug('last_slice_zb is %d', last_slice_zb)
        for disc_zb in range(self.settings.total_set_count):
            disc_title = self.disc_title(basename, set_number_zb, disc_zb)
            disc_dir = self.written_disc_directory(disc_title)
            for f in os.listdir(disc_dir):
                if f.endswith('.dar'):
                    n = self._number_from_slice_name_zb(f)
                    if n >= first_slice_zb and n <= last_slice_zb:
                        self._copy(os.path.join(disc_dir, f),
                                   os.path.join(self.settings.scratch_dir, f))
                elif parity_volume_re.match(f):
                    a, b = self._numbers_from_par_filename_zb(f)
                    if a >= first_slice_zb and b <= last_slice_zb:
                        self._copy(os.path.join(disc_dir, f),
                                   os.path.join(self.settings.scratch_dir, f))
                elif f.endswith('.par'):
                    if f in pars_hereafter:
                        self._copy(os.path.join(disc_dir, f),
                                   os.path.join(self.settings.scratch_dir, f))
        for (a,b), parfilename in zip(parity_sets_hereafter, pars_hereafter):
            if a >= first_slice_zb and b <= last_slice_zb:
                self._run('parchive', 'r', parfilename)

    def _extract(self, dir, basename, number, extension, happening):
        number = int(number)
        if number == 0:
            # dar wants the last slice but doesn't know its number
            self._fetch_some_slices(basename,
                                    *self._last_parity_set_slices_zb(basename))
        else:
            slice_name = self._slice_name(basename, number, extension)
            if os.path.exists(slice_name):
                # the first time this gets called with a real number,
                # happening is still 'init' so the hawkeyed will see
                # one of these messages before we go back to set 1
                self.log.debug('the file for slice (ob) %s already exists',
                               number)
                return
            else:
                self._fetch_some_slices(basename, number)

    _list = _extract

class TestDigits(unittest.TestCase):
    def test1(self):
        self.settings = Settings()
        self.settings.expected_data_size = 540.0
        self.settings.slices_per_disc = 500
        self.settings.disc_size = 23841
        # around 12,000 total slices
        self.assertEqual(self.settings.digits, 6)

class UsesTempScratchDir(unittest.TestCase):
    def setUp(self):
        self.settings = Settings()
        tempdir = tempfile.mkdtemp('darbrrb_test')
        self.old_tempfile_tempdir = tempfile.tempdir
        tempfile.tempdir = tempdir
        self.settings.scratch_dir = tempdir
        self.log = logging.getLogger('test code')
        self.dars_created = []
        self.par_pxx_files_created = []

    def tearDown(self):
        shutil.rmtree(self.settings.scratch_dir)
        tempfile.tempdir = self.old_tempfile_tempdir

    def mkdirp_parents(self, *names):
        for name in names:
            dir, file = os.path.split(name)
            where = self.settings.scratch_dir
            for d in dir.split(os.path.sep):
                new = os.path.join(where, d)
                if not os.path.exists(new):
                    os.mkdir(new)
                where = new

    def touch(self, *filenames):
        self.mkdirp_parents(*filenames)
        for name in filenames:
            with open(name, 'wt') as f:
                print('*', file=f)

    def mkdirp(self, *dirnames):
        self.mkdirp_parents(*dirnames)
        for name in dirnames:
            if not os.path.exists(name):
                os.mkdir(name)

    @property
    def dar_filename_format(self):
        return '{{}}.{}.dar'.format(self.settings.number_format)

    def touch_dar_file(self, basename, n):
        filename = self.dar_filename_format.format(basename,n)
        self.dars_created.append(filename)
        self.touch(filename)

    def touch_dar_files(self, basename, min, max):
        self.touch(*(self.dar_filename_format.format(basename,n)
                for n in range(min, max+1)))

    def touch_dar_file_in_output_dir(self, dir, basename, n):
        filename = self.dar_filename_format.format(basename, n)
        self.touch(os.path.join(dir, filename))

    @property
    def par_main_format(self):
        return '{{}}.{0}-{0}.par'.format(self.settings.number_format)

    @property
    def par_volume_format(self):
        return '{{}}.{0}-{0}.{{}}{{:02d}}'.format(self.settings.number_format)

    def par_volume_names(self, basename, min_, max_, count):
        possible_volume_letters = 'pqrstuvxwyz'
        for i in range(count):
            # name.p00, name.p01, ..., name.p99, name.q00, ...
            letter = possible_volume_letters[i // 100]
            number = i % 100
            pfn = self.par_volume_format.format(basename, min_, max_,
                                                letter, number)
            yield pfn

    def touch_par_files(self, basename, min_, max_, count):
        self.touch(self.par_main_format.format(basename, min_, max_))
        for pfn in self.par_volume_names(basename, min_, max_, count):
            self.par_pxx_files_created.append(pfn)
            self.touch(pfn)

    def touch_par_file_in_output_dir(self, dir, basename, min_, max_):
        fn = os.path.join(dir,
                          self.par_main_format.format(basename, min_, max_))
        #self.log.debug('touching %r', fn)
        self.touch(fn)

class TestWorkingDirectoryContextManager(unittest.TestCase):
    @patch('os.chdir')
    @patch('os.getcwd', return_value='/zart')
    def testWorkingDirectory(self, getcwd, chdir):
        with working_directory('/fnord'):
            pass
        chdir.assert_has_calls([call('/fnord'),
                                call('/zart')])

@patch('os.chdir')
@patch('os.getcwd', return_value='/zart')
@patch.object(Darbrrb, '_run')
class TestInvokeDar(UsesTempScratchDir):
    def setUp(self):
        super().setUp()
        self.d = Darbrrb(self.settings, __file__)

    def testInvokeDar(self, _run, getcwd, chdir):
        self.d.dar('-c', 'basename', '-R', '/home/bla/photos')
        self.d._run.assert_called_with(
                'dar', '-c', 'basename', '-R', '/home/bla/photos',
                '-B', os.path.join(self.settings.scratch_dir, 'darrc'))

    def testCurrentWorkingDirectory(self, _run, getcwd, chdir):
        self.d.dar('-c', 'basename', '-R', '/home/bla/photos')
        # we can only assume the chdir calls surround the _run
        chdir.assert_has_calls([
                call(self.settings.scratch_dir),
                call('/zart')])

@patch.object(Darbrrb, '_run')
@patch.object(Darbrrb, 'wait_for_empty_disc')
class TestDarbrrbFourPlusOne(UsesTempScratchDir):
    data_discs = 4
    parity_discs = 1
    slices_per_disc = 5
    pretend_free_space_MiB = (data_discs + parity_discs) * 25000

    def setUp(self):
        super().setUp()
        self.settings.data_discs = self.data_discs
        self.settings.parity_discs = self.parity_discs
        self.settings.slices_per_disc = self.slices_per_disc
        self.settings.digits = 4
        self.settings.burner_device = '/dev/zero'
        # in our tests, _create is called, as though dar were invoking this
        # script; when dar does that, it's with the scratch dir as the cwd,
        # as tested above
        with patch.object(Darbrrb, 'scratch_free_MiB',
                          return_value=self.pretend_free_space_MiB):
            self.d = Darbrrb(self.settings, __file__)
            self.d.ensure_scratch()
            self.cwd = os.getcwd()
            os.chdir(self.settings.scratch_dir)

    def tearDown(self):
        super().tearDown()
        os.chdir(self.cwd)

    def testFirstFileOfSet(self, wfed, _run):
        self.touch_dar_files('thing', 1,1)
        everything = list(os.walk(self.settings.scratch_dir))
        self.d._create('dir', 'thing', '1', 'dar', 'operating')
        everything2 = list(os.walk(self.settings.scratch_dir))
        self.assertEqual(everything, everything2)

    def testLastFileOfSet(self, wfed, _run):
        self.touch_dar_files('thing', 1,4)
        self.touch_par_files('thing', 1,4,1)
        self.d._create('dir', 'thing', '4', 'dar', 'operating')
        self.d._run.assert_any_call(
                'parchive', '-n1', 'a',
                'thing.0001-0004.par',
                'thing.0001.dar', 'thing.0002.dar',
                'thing.0003.dar', 'thing.0004.dar')
        self.assertEqual(self.d._run.call_count, 1)
        self.assertEqual(glob.glob('*.dar'), [])

    # the last set, our discs may not be full, but because we stripe files
    # across the whole set, they are likely all non-empty.
    def testLastDiscOfBackupNotEven(self, wfed, _run):
        self.touch_dar_files('thing', 13, 14)
        self.touch_par_files('thing', 13, 14, 1)
        self.d._create('dir', 'thing', '14', 'dar', 'last_slice')
        self.d._run.assert_any_call(
                'parchive', '-n1', 'a',
                'thing.0013-0014.par',
                'thing.0013.dar', 'thing.0014.dar')
        self.d._run.assert_has_calls([
            call('growisofs', '-Z', '/dev/zero', '-R', '-J',
                    '-V', 'thing-0001-001', '__disc0001'),
            call('growisofs', '-Z', '/dev/zero', '-R', '-J',
                    '-V', 'thing-0001-002', '__disc0002'),
            call('growisofs', '-Z', '/dev/zero', '-R', '-J',
                    '-V', 'thing-0001-003', '__disc0003'),
            call('growisofs', '-Z', '/dev/zero', '-R', '-J',
                    '-V', 'thing-0001-004', '__disc0004'),
            call('growisofs', '-Z', '/dev/zero', '-R', '-J',
                    '-V', 'thing-0001-005', '__disc0005'),
            ])
        self.assertEqual(self.d._run.call_count, 6)

class TestDiscTitle(unittest.TestCase):
    def setUp(self):
        self.settings = Settings()
        self.settings.digits = 4
        self.d = Darbrrb(self.settings, __file__)
        self.log = logging.getLogger(self.__class__.__name__)

    def testQuickCheck(self):
        passes = 0
        fails = 0
        tries = 100
        for qc in range(tries):
            self.settings.data_discs = random.randint(1,40)
            self.settings.parity_discs = random.randint(1,40)
            self.settings.slices_per_disc = random.randint(1,100)
            sps = self.settings.slices_per_set
            complete_sets = random.randint(1,10)
            slices_in_last_set = random.randint(1,self.settings.slices_per_disc)
            calls = []
            # sets are numbered starting with 1
            for set in range(1,complete_sets+1):
                calls.append((set, (set * sps), 'operating'))
            calls.append((set+1, ((set * sps) +
                                  random.randint(1, sps)),
                            'last_slice'))
            for set, slice, happening in calls:
                for disc in range(self.settings.total_set_count):
                    should_name = 'fnord-%04d-%03d' % (set, disc+1)
                    is_name = self.d.disc_title_for_slice_and_disc('fnord', slice, disc)
                    self.assertEqual(is_name, should_name,
                            'with {s.data_discs} data discs, ' \
                            '{s.slices_per_disc} slices per disc, ' \
                            '{cs} complete sets, {sils} slices in last set, ' \
                            'on set {set}, slice {slice}, ' \
                            'happening {happening}, calls was {calls}, ' \
                            'disc title should be ' \
                            '{should_name}, but is {is_name}'.format(
                                s=self.settings,
                                cs=complete_sets, sils=slices_in_last_set,
                                set=set, slice=slice, happening=happening,
                                calls=calls,
                                should_name=should_name, is_name=is_name))

    def testFourPlusOne(self):
        self.settings.data_discs = 4
        self.settings.parity_discs = 1
        self.settings.slices_per_disc = 11
        for set, slice, happening in (
                (1, 44, 'operating'),
                (2, 88, 'operating'),
                (3, 100, 'last_slice')):
            for disc in range(self.settings.total_set_count):
                should_name = 'fnord-%04d-%03d' % (set, disc + 1)
                is_name = self.d.disc_title_for_slice_and_disc('fnord', slice, disc)

    def testThreePlusEight(self):
        self.settings.data_discs = 3
        self.settings.parity_discs = 8
        self.settings.slices_per_disc = 20
        for set, slice, happening in ((1, 60, 'operating'), (2, 98, 'last_slice')):
            for disc in range(self.settings.total_set_count):
                should_name = 'fnord-%04d-%03d' % (set, disc + 1)
                is_name = self.d.disc_title_for_slice_and_disc('fnord', slice, disc)
                self.assertEqual(is_name, should_name)

@patch.object(Darbrrb, '_run')
@patch.object(Darbrrb, 'wait_for_empty_disc')
class TestWholeBackup(UsesTempScratchDir):
    data_discs = 4
    parity_discs = 1
    slices_per_disc = 5
    pretend_free_space_MiB = (data_discs + parity_discs) * 25000

    def setUp(self):
        super().setUp()
        self.log = logging.getLogger('test code')
        self.settings.data_discs = self.data_discs
        self.settings.parity_discs = self.parity_discs
        self.settings.slices_per_disc = self.slices_per_disc
        # all tests not written with a small disc size expect a large one.
        self.settings.disc_size_MiB = getattr(self, 'disc_size_MiB', 23841)
        self.settings.burner_device = '/dev/zero'
        # in our tests, _create is called, as though dar were invoking this
        # script; when dar does that, it's with the scratch dir as the cwd,
        # as tested above
        with patch.object(Darbrrb, 'scratch_free_MiB',
                          return_value=self.pretend_free_space_MiB):
            self.d = Darbrrb(self.settings, __file__)
            self.d.ensure_scratch()
            self.cwd = os.getcwd()
            os.chdir(self.settings.scratch_dir)
        self.discs_burned = []
        self.dar_create_slices_count = 0 # see test methods

    def tearDown(self):
        super().tearDown()
        os.chdir(self.cwd)

    def mock_dar(self, *args):
        # set self.dar_create_slices_count before you run this method
        #
        # we assume here that dar is being called to create the archive
        # args goes like dar -c basename -R root_of_files_to_save
        dir = args[-1]
        basename = args[2]
        for slice in range(self.dar_create_slices_count-1):
            # slice is zero-based but dar is one-based
            self.touch_dar_file(basename, slice+1)
            self.d._create(dir, basename, str(slice+1), 'dar', 'operating')
        slice += 1
        self.touch_dar_file(basename, slice+1)
        self.d._create(dir, basename, str(slice+1), 'dar', 'last_slice')

    def mock_parchive(self, *args):
        # expected args: parchive -n{} c outfile infile infile...
        # -n is the number of redundancy files to make
        # outfile is named {basename}.{slicemin}-{slicemax}.par
        outfile = args[3]
        nswitch = args[1]
        bn, numbers, par = outfile.split('.')
        n1, n2 = map(int, numbers.split('-'))
        count = int(nswitch.lstrip('-n'))
        self.touch_par_files(bn, n1, n2, count)

    def mock_growisofs(self, *args):
        # expected args: growisofs .... dirname
        files = list(map(os.path.basename,
                sorted(glob.glob(os.path.join(args[-1], '*')))))
        # self.log.debug('burning disc with files {}'.format(files))
        self.discs_burned.append(files)

    def mock__run(self, *args):
        # self.log.debug('args are %r', args)
        if args[0] == 'dar':
            self.mock_dar(*args)
        elif args[0] == 'parchive':
            self.mock_parchive(*args)
        elif args[0] == 'growisofs':
            self.mock_growisofs(*args)
        else:
            raise Exception('unknown command run under test', args)

    def testWholeBackup(self, wfed, _run):
        dir = 'dir'
        bn = 'whole'
        sett = self.settings
        complete_redundancy_sets = 2
        _run.side_effect = self.mock__run
        # two whole redundancy sets
        # a couple of file sets, not enough to make a redundancy set
        # a couple of odd files
        self.dar_create_slices_count = \
                sett.slices_per_disc * sett.data_discs * \
                        complete_redundancy_sets + \
                sett.slices_per_disc * sett.data_discs // 2 + \
                sett.data_discs // 2
        # for each complete or partial (at end) set of {data_discs} dar files,
        # we run parchive once
        expected_pars_run = sett.slices_per_disc * \
                        complete_redundancy_sets + \
                sett.slices_per_disc // 2 + \
                1
        # --- run code
        self.d.dar('-c', bn, '-R', '/fnord')
        max_slice = self.dar_create_slices_count
        # --- assertions
        #self.log.debug('calls:')
        #for c in self.d._run.call_args_list:
        #    self.log.debug('%r', c)
        def calls_running(executable):
            return list(x for x in self.d._run.call_args_list
                    if x[0][0] == executable)
        self.assertEqual(len(calls_running('parchive')), expected_pars_run)
        expected_discs_burned = sett.total_set_count * (
            complete_redundancy_sets + 1)
        self.assertEqual(len(calls_running('growisofs')),
                expected_discs_burned)
        unburned_dars = set(self.dars_created)
        unburned_par_volumes = set(self.par_pxx_files_created)
        for b in self.discs_burned:
            unburned_dars -= set(b)
            unburned_par_volumes -= set(b)
        self.assertEqual(len(unburned_dars), 0)
        # the last redundancy set will be incomplete
        disc = 0
        for redundancy_set in range(complete_redundancy_sets + 1):
            for disc_in_set in range(sett.total_set_count):
                b = self.discs_burned[disc]
                self.assertTrue('README.txt' in b)
                this_script = os.path.basename(__file__)
                self.log.debug('__file__ is %s', this_script)
                # this will likely be darbrrb.py - but it isn't
                # certain. see when self.d gets created above
                self.assertTrue(this_script in b)
                dars_on_b = [x for x in b if x.endswith('.dar')]
                # the code uses regexes; let's use a different way
                # here in the test for thoroughness
                par_volumes_on_b = [x for x in b
                                    if x[-4] == '.' and x[-3] in 'pqr'
                                       and x[-2] in '0123456789'
                                       and x[-1] in '0123456789']
                par_files_on_b = [x for x in b if x.endswith('.par')]
                if disc_in_set < sett.data_discs:
                    self.assertEqual(len(par_volumes_on_b), 0)
                    self.assertGreater(len(dars_on_b), 0)
                    self.assertGreater(len(par_files_on_b), 0)
                else:
                    self.assertEqual(len(dars_on_b), 0)
                    self.assertGreater(len(par_volumes_on_b), 0)
                    self.assertGreater(len(par_files_on_b), 0)
                if redundancy_set < complete_redundancy_sets:
                    # not the last set; should have not only some
                    # files, but a full complement of them
                    if disc_in_set < sett.data_discs:
                        self.assertEqual(len(par_files_on_b), sett.slices_per_disc)
                        self.assertEqual(len(dars_on_b), sett.slices_per_disc)
                    else:
                        self.assertEqual(len(par_files_on_b), sett.slices_per_disc)
                        self.assertEqual(len(par_volumes_on_b), sett.slices_per_disc)
                disc += 1

# the decorators above have made it so that TestWholeBackup is decorated. when
# we derive from it here, our ThreePlusEight class is already decorated; so we
# needn't (and mustn't) write the decorators again.
class TestWholeBackupThreePlusEight(TestWholeBackup):
    data_discs = 3
    parity_discs = 8
    slices_per_disc = 13
    pretend_free_space_MiB = (data_discs + parity_discs) * 25000

# This is just to make sure all those integer divisions and multiplications
# aren't just happening to be right.
class TestWholeBackupNineteenPlusSeven(TestWholeBackup):
    data_discs = 19
    parity_discs = 7
    slices_per_disc = 31
    pretend_free_space_MiB = (data_discs + parity_discs) * 25000

# Really small discs are useful for manual testing, where we want to
# get an idea of how fast things will run or something, but don't want
# to waste a lot of space or time doing so at scale.
class TestReallySmallDiscs(TestWholeBackup):
    data_discs = 3
    parity_discs = 2
    slices_per_disc = 50
    pretend_free_space = (data_discs + parity_discs) * 25000
    disc_size_MiB = 30

@patch.object(Darbrrb, '_run')
@patch.object(Darbrrb, '_copy')
class TestWholeRestore(UsesTempScratchDir):
    data_discs = 4
    parity_discs = 1
    slices_per_disc = 5
    pretend_free_space_MiB = (data_discs + parity_discs) * 25000

    def setUp(self):
        super().setUp()
        self.log = logging.getLogger('test code')
        self.settings.data_discs = self.data_discs
        self.settings.parity_discs = self.parity_discs
        self.settings.slices_per_disc = self.slices_per_disc
        self.settings.actually_burn = False
        # all tests not written with a small disc size expect a large one.
        self.settings.disc_size_MiB = getattr(self, 'disc_size_MiB', 23841)
        self.settings.burner_device = '/dev/zero'
        # in our tests, _create is called, as though dar were invoking this
        # script; when dar does that, it's with the scratch dir as the cwd,
        # as tested above
        with patch.object(Darbrrb, 'scratch_free_MiB',
                          return_value=self.pretend_free_space_MiB):
            self.d = Darbrrb(self.settings, __file__)
            self.d.ensure_scratch()
            self.cwd = os.getcwd()
            os.chdir(self.settings.scratch_dir)
        from hashlib import sha256
        self.basename = sha256(bytes(self.__class__.__name__, 'ascii')).hexdigest()[:8]
        self.discs_consumed = []
        self.complete_redundancy_sets = 2
        # two whole redundancy sets
        # a couple of file sets, not enough to make a redundancy set
        # a couple of odd files
        sett = self.settings
        self.dar_consume_slices_count = \
                sett.slices_per_disc * sett.data_discs * \
                        self.complete_redundancy_sets + \
                sett.slices_per_disc * sett.data_discs // 2 + \
                sett.data_discs // 2
        self.log.debug('dar_consume_slices_count is %d',
                       self.dar_consume_slices_count)
        # BWAHAHAHAA this painful thing is so the main code and the
        # test will be coded differently.
        data_set_and_disc_zb = itertools.chain(
            # complete sets
            itertools.chain.from_iterable(
                itertools.islice(zip(itertools.repeat(setno),
                                     itertools.cycle(range(sett.data_discs))),
                                 sett.slices_per_disc * sett.data_discs)
                for setno in range(self.complete_redundancy_sets)),
            # now the last set
            itertools.chain.from_iterable(
                itertools.islice(zip(itertools.repeat(setno),
                                     itertools.cycle(range(sett.data_discs))),
                                 (sett.slices_per_disc * sett.data_discs // 2 +
                                  sett.data_discs // 2))
                for setno in range(self.complete_redundancy_sets,
                                   self.complete_redundancy_sets+1)))
        data_set_and_disc_zb = list(data_set_and_disc_zb)
        for slice_zb, (setno_zb, disc_zb) in zip(
                range(self.dar_consume_slices_count),
                data_set_and_disc_zb):
            # disc_title gets tested above, so we'll just use it here
            disc_title = self.d.disc_title(self.basename, setno_zb, disc_zb)
            self.touch_dar_file_in_output_dir(disc_title, self.basename,
                                              slice_zb+1)
            # par files go on every data disc
            self.touch_par_file_in_output_dir(disc_title, self.basename,
                                              (slice_zb - disc_zb)+1,
                                              min(((slice_zb - disc_zb)+1 +
                                                   self.data_discs-1),
                                                  self.dar_consume_slices_count))
        parity_set_and_disc_zb = itertools.chain(
            # complete sets
            itertools.chain.from_iterable(
                itertools.islice(zip(itertools.repeat(setno),
                                     itertools.cycle(range(sett.data_discs,
                                                           (sett.data_discs +
                                                            sett.parity_discs)))),
                                 sett.slices_per_disc * sett.parity_discs)
                for setno in range(self.complete_redundancy_sets+2)))
            # now the last set
            # itertools.chain.from_iterable(
            #     itertools.islice(zip(itertools.repeat(setno),
            #                          itertools.cycle(range(sett.data_discs,
            #                                                (sett.data_discs +
            #                                                 sett.parity_discs)))),
            #                      (sett.slices_per_disc * sett.parity_discs // 2 +
            #                       sett.parity_discs // 2))
            #     for setno in range(self.complete_redundancy_sets,
            #                        self.complete_redundancy_sets+1)))
        parity_set_and_disc_zb = list(parity_set_and_disc_zb)
        mins_and_maxes = itertools.chain.from_iterable(
            map(itertools.repeat, zip(
                range(0, self.dar_consume_slices_count, self.data_discs),
                itertools.chain(
                    range(self.data_discs-1, self.dar_consume_slices_count,
                          self.data_discs),
                    itertools.repeat(self.dar_consume_slices_count-1))),
                           itertools.repeat(self.parity_discs)))
        mins_and_maxes = list(mins_and_maxes)
        for (min_zb, max_zb), (setno_zb, disc_zb) in zip(
                mins_and_maxes,
                parity_set_and_disc_zb):
            disc_title = self.d.disc_title(self.basename, setno_zb, disc_zb)
            self.touch_par_file_in_output_dir(disc_title, self.basename,
                                              min_zb+1, max_zb+1)
        # cheat: we don't touch the par volume files. only the real
        # parchive actually needs them!
#        os.system('bash')

    def tearDown(self):
        super().tearDown()
        os.chdir(self.cwd)

    def mock_dar(self, *args):
        # set self.dar_consume_slices_count before you run this method
        #
        # we assume here that dar is being called to extract the archive
        # args goes like dar -x basename -R root_of_files_to_save
        dir = args[-1]
        basename = args[2]
        # get last slice
        self.d._extract(dir, basename, '0', 'dar', 'init')
        self.d._extract(dir, basename, self.dar_consume_slices_count, 'dar', 'init')
        for slice in range(self.dar_consume_slices_count):
            # slice is zero-based but dar is one-based
            self.d._extract(dir, basename, str(slice+1), 'dar', 'operating')

    def mock_parchive(self, *args):
        # expected args: parchive r parfile
        # parfile is named {basename}.{slicemin}-{slicemax}.par
        parfile = args[2]
        # put logic here in future to mock one or more of the files
        # being invalid or missing

    def mock__run(self, *args):
        self.log.debug('args are %r', args)
        if args[0] == 'dar':
            self.mock_dar(*args)
        elif args[0] == 'parchive':
            self.mock_parchive(*args)
        else:
            raise Exception('unknown command run under test', args)

    def testWholeRestore(self, _copy, _run):
        dir = 'dir'
        _run.side_effect = self.mock__run
        _copy.side_effect = shutil.copyfile
        # we run parchive once to get the last slice. then,
        #
        # for each complete or partial (at end) set of {data_discs} dar files,
        # we run parchive once
        sett = self.settings
        expected_pars_run = (1 +
                             (sett.slices_per_disc *
                              self.complete_redundancy_sets) +
                             (sett.slices_per_disc // 2 + 1))
        # --- run code
        self.d.dar('-x', self.basename, '-R', '/fnord')
        # --- assertions
        self.log.debug('calls:')
        for c in self.d._run.call_args_list:
            self.log.debug('%r', c)
        def calls_running(executable):
            return list(x for x in self.d._run.call_args_list
                    if x[0][0] == executable)
        #self.log.debug(calls_running('parchive'))
        unique_basenames_copied = sorted(list(set(
            [os.path.basename(c[0][0]) for c in self.d._copy.call_args_list])))
        self.log.debug('unique basenames copied: %r', unique_basenames_copied)
        self.assertEqual(len([b for b in unique_basenames_copied
                              if b.endswith('.par')]),
                         ((self.dar_consume_slices_count +
                           self.settings.data_discs-1) //
                          self.settings.data_discs),
                         'unexpected number of pars copied')
        self.assertEqual(len(calls_running('parchive')), expected_pars_run)
        # not tested so far:
        # * only the files for one set are in the scratch dir at once

class TestWholeRestoreThreePlusEight(TestWholeRestore):
    data_discs = 3
    parity_discs = 8
    slices_per_disc = 13
    pretend_free_space_MiB = (data_discs + parity_discs) * 25000

class TestWholeRestoreNineteenPlusSeven(TestWholeRestore):
    data_discs = 19
    parity_discs = 7
    slices_per_disc = 31
    pretend_free_space_MiB = (data_discs + parity_discs) * 25000

@patch.object(Darbrrb, '_run')
class TestPartialRestore(TestWholeRestore):
    def setUp(self):
        super().setUp()
        # now, unlike the full restore, we will not ask for every
        # slice. just some toward the end of the first set, and into
        # the second set.
        s = self.settings
        # the first fetch will do 9% of the slices; make it fetch
        # twice from the first set
        self.first_slice_dar_requests = int(s.slices_per_set * (1 - 0.14))
        # and only once from the beginning of the second set
        self.last_slice_dar_requests = int(s.slices_per_set * (1 + 0.08))

    def mock_dar(self, *args):
        # set self.dar_consume_slices_count before you run this method
        #
        # we assume here that dar is being called to extract the archive
        # args goes like dar -x basename -R root_of_files_to_save
        dir = args[-1]
        basename = args[2]
        # get last slice
        self.d._extract(dir, basename, '0', 'dar', 'init')
        self.d._extract(dir, basename, self.dar_consume_slices_count, 'dar', 'init')
        for slice in range(self.first_slice_dar_requests,
                           self.last_slice_dar_requests + 1):
            # slice is zero-based but dar is one-based
            self.d._extract(dir, basename, str(slice+1), 'dar', 'operating')

    # we are testing a partial restore, but deriving from the whole
    # restore class. if we do not call this method testWholeRestore,
    # the superclass's test method will remain.
    def testWholeRestore(self, _run):
        dir = 'dir'
        _run.side_effect = self.mock__run
        # we run parchive once to get the last slice. then,
        #
        # for each complete or partial (at end) set of {data_discs} dar files,
        # we run parchive once
        sett = self.settings
        fsdrzb = self.first_slice_dar_requests - 1
        lsdrzb = self.last_slice_dar_requests - 1
        first_slice_fetched_zb = fsdrzb - fsdrzb % sett.data_discs
        # this simple math below only works because we are certainly
        # not running into the last parity set; that set can be incomplete.
        last_slice_fetched_zb = (lsdrzb - lsdrzb % sett.data_discs +
                                 sett.data_discs - 1)
        complete_parity_sets = (last_slice_fetched_zb + 1 -
                                first_slice_fetched_zb) // sett.data_discs
        expected_pars_run = (1 + # for the last slice
                             complete_parity_sets)
        # --- run code
        self.d.dar('-x', self.basename, '-R', '/fnord', 'some-file')
        # --- assertions
        self.log.debug('calls:')
        for c in self.d._run.call_args_list:
            self.log.debug('%r', c)
        def calls_running(executable):
            return list(x for x in self.d._run.call_args_list
                    if x[0][0] == executable)
        self.log.debug(calls_running('parchive'))
        self.assertEqual(len(calls_running('parchive')), expected_pars_run)
        # not tested so far:
        # * only the files for one set are in the scratch dir at once

class TestPartialRestoreThreePlusEight(TestPartialRestore):
    data_discs = 3
    parity_discs = 8
    slices_per_disc = 13
    pretend_free_space_MiB = (data_discs + parity_discs) * 25000

class TestPartialRestoreNineteenPlusSeven(TestPartialRestore):
    data_discs = 19
    parity_discs = 7
    slices_per_disc = 31
    pretend_free_space_MiB = (data_discs + parity_discs) * 25000

if __name__ == '__main__':
    s = Settings()
    if len(sys.argv) < 2:
        usage(s)
        sys.exit(1)
    opts, remaining = getopt.getopt(sys.argv[1:], 'hvnt', ['help'])
    loglevel = logging.WARNING
    testing = False
    for o, v in opts:
        if o == '-h' or o == '--help':
            usage(s)
            sys.exit(1)
        elif o == '-v':
            loglevel -= 10
        elif o == '-n':
            s.actually_burn = False
            # warn user
            from time import sleep
            for i in range(25):
                for j in range(4):
                    print('not actually burning', end=' * ')
                print('\n')
            sleep(5)
        elif o == '-t':
            loglevel = logging.DEBUG
            testing = True
        else:
            raise Exception('unknown switch {}'.format(o))
    # style only influences how format is interpreted, not also how values are
    # interpolated into log messages. source: Python 3.2
    # logging/__init__.py:317, LogRecord class, getMessage method.
    fmt = logging.Formatter(style='{', fmt='darbrrb {process} '
                        '[{asctime}] {name}: {message}')
    stream = logging.StreamHandler(sys.stderr)
    stream.setFormatter(fmt)
    root_logger = logging.getLogger()
    root_logger.addHandler(stream)
    root_logger.setLevel(loglevel)
    # dar expects its stdin and stdout to be a terminal so we will
    # make a log file for our messages rather than depend on
    # redirection
    logfile = logging.FileHandler('darbrrb.log.{}'.format(os.getpid()))
    logfile.setFormatter(fmt)
    root_logger.addHandler(logfile)
    log = logging.getLogger('__main__'.format(os.getpid()))
    log.debug('called with args %r', sys.argv)
    if testing:
        # strip off switches: they are not for unittest.main
        sys.argv = [sys.argv[0]] + remaining
        sys.exit(unittest.main())
    d = Darbrrb(s, __file__, opts)
    try:
        if remaining[0] == 'dar':
            os.environ['DARBRRB_ORIGINAL_ARGV'] = base64.b64encode(
                pickle.dumps(sys.argv, protocol=0)).decode('UTF-8')
            d.ensure_scratch()
            d.dar(*remaining[1:])
        elif remaining[0] == '_create':
            d._create(*remaining[1:])
        elif remaining[0] == '_extract':
            d._extract(*remaining[1:])
        elif remaining[0] == '_list':
            d._list(*remaining[1:])
        else:
            raise Exception("unknown subcommand", remaining)
        log.debug('execution ended without exception')
    except Exception as e:
        log.exception('execution ended abnormally')
        raise


import cv

cv.NamedWindow("w1", cv.CV_WINDOW_AUTOSIZE)
capture = cv.CaptureFromCAM(0)

def repeat():
    while True:
        frame = cv.QueryFrame(capture)
        cv.ShowImage("w1", frame)
        cv.WaitKey(10)

repeat()
cv.WaitKey(0)


#!/usr/bin/env python3
# coding=utf-8

import rospy
from math import *
from sensor_msgs.msg import Image
import cv2, cv_bridge
import numpy as np
from geometry_msgs.msg import Twist
from yolo_new.msg import position_color as PositionMsg
from yolo_new.msg import color_ik_result_new as color_ik_result_Msg
from std_msgs.msg import Int8
from std_msgs.msg import String as StringMsg
from yolov5_ros_msgs.msg import BoundingBox,BoundingBoxes
from yolo_new.msg import Flag,Serial_RT
import serial   #导入串口通信库
from time import sleep

ser = serial.Serial()

class Serial_COM:
	def __init__(self):

		self.com_sub = rospy.Subscriber("/Com_pub", Serial_RT, self.com_callback)#订阅识别类别话题
		self.flagPublisher = rospy.Publisher('/Flag_pub', Flag, queue_size=10) #发布色块的位置（原始数据）

		self.port_open_recv()		#实机测试放出来!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

	def com_callback(self,msg):
		rospy.loginfo("~~~~通信回调收到: %s,%d,%d",msg.sendClass, msg.count,msg.ONum)
		if msg.close == "ok":
			self.send('1P')
			rospy.loginfo("~~~~通信关闭盖子: %s",msg.sendClass)
			sleep(1)
		else:
			self.send_message(msg.count,msg.sendClass,msg.ONum)

	# # 超声波投入发布flag（暂时不用）
	# def input_pubflag(self):	#超声波判断投入发布flag
	# 	ismoving = 0
	# 	isputing = 1
	# 	singleOK = 1
    # 	# count  预留，为超声波计数
	# 	flagMSG = Flag(ismoving,isputing,singleOK)
	# 	self.flagPublisher.publish(flagMSG)
	# 	isputing = 0

	def single_pubflag(self):	# 刷子完成分拣后，接收到串口消息时进行发送
		ismoving = 0
		isputing = 0
		singleOK = 1
		OverLoad = "none"
		flagMSG = Flag(ismoving,isputing,singleOK,OverLoad)
		self.flagPublisher.publish(flagMSG)

	def overload_pubflag(self,sort):
		ismoving = 1
		isputing = 0
		singleOK = 1
		OverLoad = sort	# ABCD
		flagMSG = Flag(ismoving,isputing,singleOK,OverLoad)
		self.flagPublisher.publish(flagMSG)

	def send_message(self,bcount,bclass,bONum):
		# rospy.loginfo("send!!!!!!!!!!!!!!!!!!!!!!!!!!11")
		if bONum == 1:
			if bcount == 1:
				self.send('0A')
				rospy.loginfo("0A")
			elif bcount >1:
				self.send('1A')
				rospy.loginfo("1A")
		elif bONum == 2:
			if bcount == 1:
				self.send('0B')
				rospy.loginfo("0B")
			elif bcount >1:
				self.send('1B')
				rospy.loginfo("1B")
		elif bONum == 3:
			if bcount == 1:
				self.send('0C')
				rospy.loginfo("0C")
			elif bcount >1:
				self.send('1C')
				rospy.loginfo("1C")
		elif bONum == 4:
			if bcount == 1:
				self.send('0D')
				rospy.loginfo("0D")
			elif bcount >1:
				self.send('1D')
				rospy.loginfo("1D")
		elif bONum == 5:
			if bcount == 1:
				self.send('0E')
				rospy.loginfo("0E")
			elif bcount >1:
				self.send('1E')
				rospy.loginfo("1E")
		elif bONum == 6:
			if bcount == 1:
				self.send('0F')
				rospy.loginfo("0F")
			elif bcount >1:
				self.send('1F')
				rospy.loginfo("1F")
		elif bONum == 7:
			if bcount == 1:
				self.send('0G')
				rospy.loginfo("0G")
			elif bcount >1:
				self.send('1G')
				rospy.loginfo("1G")
		elif bONum == 8:
			if bcount == 1:
				self.send('0H')
				rospy.loginfo("0H")
			elif bcount >1:
				self.send('1H')
				rospy.loginfo("1H")
		elif bONum == 9:
			if bcount == 1:
				self.send('0I')
				rospy.loginfo("0I")
			elif bcount >1:
				self.send('1I')
				rospy.loginfo("1I")
		else :
			rospy.loginfo("未发送！！！")

	def port_open_recv(self):#对串口的参数进行配置
		ser.port='/dev/ttyACM0'
		ser.baudrate=9600
		ser.bytesize=8
		ser.stopbits=1
		ser.parity="N"#奇偶校验位
		ser.open()
		if(ser.isOpen()):
			rospy.loginfo("串口打开成功！")
		else:
			rospy.loginfo("串口打开失败！")
	#isOpen()函数来查看串口的开闭状态

	def port_close(self):
		ser.close()
		if(ser.isOpen()):
			rospy.loginfo("串口关闭失败！")
		else:
			rospy.loginfo("串口关闭成功！")

	def send(self,send_data):
		if(ser.isOpen()):
			ser.write(send_data.encode())#编码  'utf-8'
			rospy.loginfo("发送成功! %s",send_data)
		else:
			rospy.loginfo("发送失败！")

rospy.init_node("Serial_COM")
rt_serial = Serial_COM()
# 实际调试放出来！！！！！！！！！！！！！！！！！！！！！！！！
while 1:
	strRead = ser.read().decode()#"utf-8"
	if strRead:
		rospy.loginfo("串口读取到：%s",strRead)
		if strRead == 'A' or strRead == 'B' or strRead == 'C' or strRead == 'D':
			rospy.loginfo("满载返包！")
			rt_serial.overload_pubflag(strRead)
		elif strRead == 'S':
			rospy.loginfo("单目标返包！")
			rt_serial.single_pubflag()
rospy.spin()

# 测试用！！！！！！！！！！！！！！！！！！！！！！！！！！！！
# rospy.spin()


# In[ ]:

discord_token = 'tokenhere' #@param {type:"string"}

# ##Module installation
# this will install all the necessary modules

# In[ ]:

# ##Download and load GPT NEO model.
# It will take a little bit

# In[3]:

from transformers import GPTNeoForCausalLM, GPT2Tokenizer

model = GPTNeoForCausalLM.from_pretrained('EleutherAI/gpt-neo-1.3B')
tokenizer = GPT2Tokenizer.from_pretrained('EleutherAI/gpt-neo-1.3B')

# add padding token to tokenizer
tokenizer.add_special_tokens({'pad_token': '[PAD]'})

# set padding token id to the id of the padding token
model.config.pad_token_id = tokenizer.pad_token_id
model.cuda()

# In[4]:

modes = dict()

modes['chat'] = {'prompt' : 'model\n\n',
                           'partner' : 'Partner: ',
                           'ai' : 'Humoid: ',
                            'end' : '\n'}

mode = modes['chat']

# In[5]:

# ##Discord bot

# In[1]:

import discord
from discord.ext import commands
import nest_asyncio
import asyncio
import time
import os

intents = discord.Intents.default()
intents.typing = False
intents.presences = False

bot = commands.Bot(command_prefix='!', intents=intents)

class TridequeEntry:
    def __init__(self, matrix=None):
        if matrix is None:
            self.matrix = [["" for _ in range(13)] for _ in range(13)]
        else:
            self.matrix = matrix

    def update_entry(self, x, y, value):
        self.matrix[x][y] = value

    def get_value(self, x, y):
        return self.matrix[x][y]

trideque_entry = TridequeEntry()

def generate_chunks(prompt, chunk_size=1500):
    words = prompt.split()
    return [' '.join(words[i:i + chunk_size]) for i in range(0, len(words), chunk_size)]

async def gpt3_generate(chunk, max_length=2000, time_limit=50.0):
    start_time = time.time()

    async def generate_response(chunk):
        inputs = tokenizer.encode(chunk, return_tensors='pt', truncation=True, max_length=512)
        inputs = inputs.cuda()
        attention_mask = inputs.ne(tokenizer.pad_token_id).float()
        outputs = model.generate(inputs, max_length=max_length, do_sample=True, max_time=time_limit, attention_mask=attention_mask)
        return tokenizer.decode(outputs[0])

    response = await generate_response(chunk)
    end_time = time.time()

    return response, end_time - start_time

async def write_to_buffer_file(response_text):
    buffer_file = "buffer.txt"
    with open(buffer_file, "w") as file:
        file.write(response_text)
    return buffer_file

async def read_from_buffer_file(buffer_file, chunk_size=1800):
    with open(buffer_file, "r") as file:
        while True:
            chunk = file.read(chunk_size)
            if not chunk:
                break
            yield chunk
    os.remove(buffer_file)

async def send_chunks(ctx, prompt_chunks, repeat_count=-1):
    total_time = 0.0
    repetition = 0
    while repeat_count == -1 or repetition < repeat_count:
        for chunk in prompt_chunks:
            gpt3_response, response_time = await gpt3_generate(chunk)
            total_time += response_time

            buffer_file = await write_to_buffer_file(gpt3_response)

            async for response_part in read_from_buffer_file(buffer_file):
                await asyncio.sleep(0.5)
                await ctx.send(response_part)  # Change this line to send multiple messages

        repetition += 1

    await ctx.send(f"Total response time: {total_time:.2f} seconds.")

@bot.event
async def on_ready():
        print(f'Logged in as {bot.user.name} (ID: {bot.user.id})')

@bot.command()
async def trideque(ctx, *, user_input):
    await ctx.send('The Matrix is loading, Robotic Reply Generating, please wait...')
    prompt_chunks = generate_chunks(user_input)
    await send_chunks(ctx, prompt_chunks)

nest_asyncio.apply()

# Replace 'your_bot_token' with your actual bot token from the Discord Developer Portal
bot.run('tokenhere')

# In[ ]:


import os
import tensorflow as tf
import dataset
import numpy as np
import matplotlib.pyplot as plt
import math
import yaml
from django.conf import settings
base_dir = os.path.dirname(__file__)
root_dir = settings.CLASSIFIED_SETTING['app']['root']
import scipy.misc

def visual_network(modelConfig,network):
    # load config
    config = yaml.safe_load(open(os.path.join(base_dir, "config.yml")))
    # modelConfig = yaml.safe_load(
    #     open(os.path.join(root_dir, config['training']['output_training_path'], network['model']['configPath'],
    #                       "config.yml")))

    # create graph
    sess = tf.Session()
    saver = tf.train.import_meta_graph(os.path.join(root_dir,
                                                    config['training']['output_training_path'],
                                                    network['model']['name']))
    saver.restore(sess, tf.train.latest_checkpoint(os.path.join(root_dir,
                                                                config['training']['output_training_path'],
                                                                network['model']['path'], './')))
    graph = tf.get_default_graph()

    # read images
    # test_path = config['visualizing']['path']  # constant value
    test_path = os.path.join(root_dir, config['visualizing']['path'])
    folder = config['visualizing']['folder']  # constant value
    num_channels = config['training']['num_channels']  # constant value
    img_size = modelConfig['network']['img_size']

    img_size_flat = img_size * img_size * num_channels
    test_images, test_ids, test_label = dataset.read_test_set(test_path, img_size, folder)

    # get tensor from graph
    x = graph.get_tensor_by_name(config['tensor_name']['input_x'] + ":0")
    x_batch = test_images.reshape(len(test_images), img_size_flat)
    y_true = graph.get_tensor_by_name(config['tensor_name']['input_y_true'] + ":0")
    y_test_images = test_label

    # create dict to runing in graph
    y_test_images = np.zeros((len(test_images), network['data']['numClasses']))
    feed_dict_testing = {x: x_batch, y_true: y_test_images}

    def visualitzing():
        return getActivations(graph.get_tensor_by_name(network['layers']['Names'] + ":0"), feed_dict_testing)

    def getActivations(layer, dict):
        units = sess.run(layer, feed_dict=dict)
        # return plotNNFilter(units)
        return imagesNNFilter(units)

    def plotNNFilter(units):
        images = []
        filters = units.shape[3]
        plt.figure(3, figsize=(20, 20))
        n_columns = 10
        n_rows = math.ceil(filters / n_columns) + 1
        for j in xrange(0, filters, network['return']['steps']):
            plt.subplot(n_rows, n_columns, j + 1)
            plt.title('Filter ' + str(j))
            plt.imshow(units[0, :, :, j], interpolation="nearest", cmap="gray")
        plt.show()
        return []

    def imagesNNFilter(units):
        images = []
        images_name = []
        filters = units.shape[3]
        for j in xrange(0, filters, network['return']['steps']):
            images.append(units[0, :, :, j])

            image_name = "outfile{0}{1}.jpg".format(j,network['layers']['Names'])
            images_name.append(image_name)
            image_path = os.path.join(root_dir, config['visualizing']['path'], "visual_gallery",image_name)
            scipy.misc.imsave(image_path, units[0, :, :, j])
        return images_name;

    return visualitzing()

#
# with open(os.path.join(base_dir, 'visualising.json')) as data_file:
#     json_file = json.load(data_file)
#     data_file.close()
#     print len(visual_network(json_file))


# coding=utf-8
import matplotlib

matplotlib.use("Agg")
import matplotlib.pyplot as plt
import argparse
import numpy as np
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D, UpSampling2D, BatchNormalization, Reshape, Permute, Activation, Input
from keras.utils.np_utils import to_categorical
from keras.preprocessing.image import img_to_array
from keras.callbacks import ModelCheckpoint
from sklearn.preprocessing import LabelEncoder
from keras.models import Model
from keras.layers.merge import concatenate
from PIL import Image
import matplotlib.pyplot as plt
import cv2
import random
import os
from tqdm import tqdm

from keras.models import *
from keras.layers import *
from keras.optimizers import *

from keras import backend as K
# K.set_image_dim_ordering('th')
K.set_image_dim_ordering('tf')

os.environ["CUDA_VISIBLE_DEVICES"] = "2"
seed = 7
np.random.seed(seed)

img_w = 256
img_h = 256

n_label = 1+1

"""for roads"""
# model_save_path = '../../data/models/unet_roads.h5'
# train_data_path = '../../data/traindata/binary/roads/'

"""for buildings"""
model_save_path = '../../data/models/unet_buildings_onehot.h5'
train_data_path = '../../data/traindata/all/binary/buildings/'

def load_img(path, grayscale=False):
    if grayscale:
        img = cv2.imread(path, cv2.IMREAD_GRAYSCALE)
    else:
        img = cv2.imread(path)
        img = np.array(img, dtype="float") / 255.0  # MY image preprocessing
    return img

"""get the train file name and divide to train and val parts"""
def get_train_val(val_rate=0.25):
    train_url = []
    train_set = []
    val_set = []
    for pic in os.listdir(train_data_path + 'src'):
        train_url.append(pic)
    random.shuffle(train_url)
    total_num = len(train_url)
    val_num = int(val_rate * total_num)
    for i in range(len(train_url)):
        if i < val_num:
            val_set.append(train_url[i])
        else:
            train_set.append(train_url[i])
    return train_set, val_set

# data for training
def generateData(batch_size, data=[]):
    # print 'generateData...'
    while True:
        train_data = []
        train_label = []
        batch = 0
        for i in (range(len(data))):
            url = data[i]
            batch += 1
            img = load_img(train_data_path + 'src/' + url)

            # Adapt dim_ordering automatically
            img = img_to_array(img)
            train_data.append(img)
            label = load_img(train_data_path + 'label/' + url, grayscale=True)
            label = img_to_array(label)
            train_label.append(label)
            if batch % batch_size == 0:
                # print 'get enough bacth!\n'
                train_data = np.array(train_data)
                train_label = np.array(train_label)
                train_label = to_categorical(train_label, num_classes=n_label)
                train_label = train_label.reshape((batch_size, img_w * img_h, n_label))
                yield (train_data, train_label)
                train_data = []
                train_label = []
                batch = 0

# data for validation
def generateValidData(batch_size, data=[]):
    # print 'generateValidData...'
    while True:
        valid_data = []
        valid_label = []
        batch = 0
        for i in (range(len(data))):
            url = data[i]
            batch += 1
            img = load_img(train_data_path + 'src/' + url)

            # Adapt dim_ordering automatically
            img = img_to_array(img)
            valid_data.append(img)
            label = load_img(train_data_path + 'label/' + url, grayscale=True)
            label = img_to_array(label)
            valid_label.append(label)
            if batch % batch_size == 0:
                valid_data = np.array(valid_data)
                valid_label = np.array(valid_label)
                valid_label = to_categorical(valid_label, num_classes=n_label)
                valid_label = valid_label.reshape((batch_size, img_w * img_h, n_label))
                yield (valid_data, valid_label)
                valid_data = []
                valid_label = []
                batch = 0

def unet():
    # inputs = Input((3, img_w, img_h))  # channels_first
    inputs = Input((img_w, img_h, 3))

    conv1 = Conv2D(32, (3, 3), activation="relu", padding="same")(inputs)
    conv1 = Conv2D(32, (3, 3), activation="relu", padding="same")(conv1)
    pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)

    conv2 = Conv2D(64, (3, 3), activation="relu", padding="same")(pool1)
    conv2 = Conv2D(64, (3, 3), activation="relu", padding="same")(conv2)
    pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)

    conv3 = Conv2D(128, (3, 3), activation="relu", padding="same")(pool2)
    conv3 = Conv2D(128, (3, 3), activation="relu", padding="same")(conv3)
    pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)

    conv4 = Conv2D(256, (3, 3), activation="relu", padding="same")(pool3)
    conv4 = Conv2D(256, (3, 3), activation="relu", padding="same")(conv4)
    # pool4 = MaxPooling2D(pool_size=(2, 2))(conv4)
    drop4 = Dropout(0.5)(conv4) # add 20180621
    pool4 = MaxPooling2D(pool_size=(2, 2))(drop4)

    conv5 = Conv2D(512, (3, 3), activation="relu", padding="same")(pool4)
    conv5 = Conv2D(512, (3, 3), activation="relu", padding="same")(conv5)
    drop5 = Dropout(0.5)(conv5)

    # up6 = concatenate([UpSampling2D(size=(2, 2))(drop5), conv4], axis=1)
    up6 = concatenate([UpSampling2D(size=(2, 2))(drop5), conv4], axis=3)
    conv6 = Conv2D(256, (3, 3), activation="relu", padding="same")(up6)
    conv6 = Conv2D(256, (3, 3), activation="relu", padding="same")(conv6)

    # up7 = concatenate([UpSampling2D(size=(2, 2))(conv6), conv3], axis=1)
    up7 = concatenate([UpSampling2D(size=(2, 2))(conv6), conv3], axis=3)
    conv7 = Conv2D(128, (3, 3), activation="relu", padding="same")(up7)
    conv7 = Conv2D(128, (3, 3), activation="relu", padding="same")(conv7)

    # up8 = concatenate([UpSampling2D(size=(2, 2))(conv7), conv2], axis=1)
    up8 = concatenate([UpSampling2D(size=(2, 2))(conv7), conv2], axis=3)
    conv8 = Conv2D(64, (3, 3), activation="relu", padding="same")(up8)
    conv8 = Conv2D(64, (3, 3), activation="relu", padding="same")(conv8)

    # up9 = concatenate([UpSampling2D(size=(2, 2))(conv8), conv1], axis=1)
    up9 = concatenate([UpSampling2D(size=(2, 2))(conv8), conv1], axis=3)
    conv9 = Conv2D(32, (3, 3), activation="relu", padding="same")(up9)
    conv9 = Conv2D(32, (3, 3), activation="relu", padding="same")(conv9)

    conv10 = Conv2D(n_label, (1, 1), activation="sigmoid")(conv9)
    conv10 = Reshape((img_w * img_h, n_label))(conv10)  # 4D(bath_size, img_w*img_h, n_label)

    model = Model(inputs=inputs, outputs=conv10)
    model.compile(optimizer='Adam', loss='binary_crossentropy', metrics=['accuracy'])
    return model

""" For tes multiGPU model"""
import keras
from keras.utils import multi_gpu_model
class CustomModelCheckpoint(keras.callbacks.Callback):

    def __init__(self,  path):
        # self.model = model
        self.path = path
        self.best_loss = np.inf

    def on_epoch_end(self, epoch, logs=None):
        val_loss = logs['val_loss']
        if val_loss < self.best_loss:
            print("\nValidation loss decreased from {} to {}, saving model".format(self.best_loss, val_loss))

        self.model.save_weights(self.path, overwrite=True)

        self.best_loss = val_loss

def train():
    EPOCHS = 10  # should be 10 or bigger number
    BS = 16

    model = unet()

    """test the model fastly but can only train one epoch, AND it does not work finally ^_^"""
    # model = multi_gpu_model(model, gpus=4)
    # model.compile(optimizer=Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
    ##### modelcheck = [CustomModelCheckpoint('./data/models/unet_fff.h5')]

    modelcheck = ModelCheckpoint(model_save_path, monitor='val_acc', save_best_only=True, mode='max')
    callable = [modelcheck]
    train_set, val_set = get_train_val()
    train_numb = len(train_set)
    valid_numb = len(val_set)
    print ("the number of train data is", train_numb)
    print ("the number of val data is", valid_numb)
    H = model.fit_generator(generator=generateData(BS, train_set), steps_per_epoch=train_numb // BS, epochs=EPOCHS,
                            verbose=1,
                            validation_data=generateValidData(BS, val_set), validation_steps=valid_numb // BS,
                            callbacks=callable, max_q_size=1)

    #plot the training loss and accuracy
    plt.style.use("ggplot")
    plt.figure()
    N = EPOCHS
    plt.plot(np.arange(0, N), H.history["loss"], label="train_loss")
    plt.plot(np.arange(0, N), H.history["val_loss"], label="val_loss")
    plt.plot(np.arange(0, N), H.history["acc"], label="train_acc")
    plt.plot(np.arange(0, N), H.history["val_acc"], label="val_acc")
    plt.title("Training Loss and Accuracy on U-Net Satellite Seg")
    plt.xlabel("Epoch #")
    plt.ylabel("Loss/Accuracy")
    plt.legend(loc="lower left")
    plt.savefig('../../data/models/unet_binary_train.png')

"""
only test
"""
window_size=256

def unet_predict(image,model):
    stride = window_size

    h, w, _ = image.shape
    print('h,w:', h, w)
    padding_h = (h // stride + 1) * stride
    padding_w = (w // stride + 1) * stride
    padding_img = np.zeros((padding_h, padding_w, 3))
    padding_img[0:h, 0:w, :] = image[:, :, :]

    padding_img = img_to_array(padding_img)

    mask_whole = np.zeros((padding_h, padding_w), dtype=np.float32)
    for i in list(range(padding_h // stride)):
        for j in list(range(padding_w // stride)):
            crop = padding_img[i * stride:i * stride + window_size, j * stride:j * stride + window_size, :3]

            crop = np.expand_dims(crop, axis=0)
            print('crop:{}'.format(crop.shape))

            # pred = model.predict_classes(crop, verbose=2)
            # pred = model.predict(crop, verbose=2)
            pred = model.predict(crop, verbose=2)
            pred = np.argmax(pred,axis=2)  #for one hot encoding

            pred = pred.reshape(256, 256)
            print(np.unique(pred))

            mask_whole[i * stride:i * stride + window_size, j * stride:j * stride + window_size] = pred[:, :]

    # outputresult = mask_whole[0:h, 0:w] * 255.0
    outputresult =mask_whole[0:h,0:w]
    outputresult = outputresult.astype(np.uint8)

    plt.imshow(outputresult, cmap='gray')
    plt.title("Original predicted result")
    plt.show()
    cv2.imwrite('../../data/predict/test_onehot.png',outputresult*255)
    return outputresult

if __name__ == '__main__':

    if not os.path.isdir(train_data_path):
        print ("train data does not exist in the path:\n {}".format(train_data_path))

    train()

    # print("test ....................predict by unet one hot encoding .....\n")
    # img_path = '../../data/test/1.png'
    # import sys
    #
    # if not os.path.isfile(img_path):
    #     print("no file: {}".forma(img_path))
    #     sys.exit(-1)
    #
    # input_img = cv2.imread(img_path)
    # input_img = np.array(input_img, dtype="float") / 255.0  # must do it
    #
    # new_model = load_model(model_save_path)
    #
    # unet_predict(input_img, new_model)


import pygame
from random import randint, random
pygame.init()
pygame.mixer.init()

pygame.mixer.music.load("Stuff/Music/music.ogg")
pygame.mixer.music.play(-1)
pygame.mixer.music.set_volume(0.5)

#create screen
screen = pygame.display.set_mode((1000, 800))
pygame.display.set_caption('Flappy by Motus')
timer = pygame.time.Clock()
icon = pygame.image.load('Stuff/image/wbird.png')
pygame.display.set_icon(icon)

#game dict:
bird = pygame.transform.scale(pygame.image.load('Stuff/image/bird.png'), (100,80))
bg = pygame.transform.scale(pygame.image.load('Stuff/image/bg.jpg'), (1000, 800))
BLACK = (0, 0, 0)
GREEN = (30,89,69)
BLUE = (176,183,198)
wing = pygame.mixer.Sound('Stuff/Music/wing.wav')
wing.set_volume(0.2)
point = pygame.mixer.Sound('Stuff/Music/point.wav')
point.set_volume(0.2)
die = pygame.mixer.Sound('Stuff/Music/die.wav')
die.set_volume(0.3)

#game varials:
play = True
score = 0
speed = 7
y_speed = 10
bird_x, bird_y = (100, 400)
FPS = 60
gravity = 0.3
trubs = [[[300, 0, 60, 400], [300, 600, 60 , 770]], [[800, -100, 60, 400], [800, 500, 60 , 770]]]
y_change = 0
gameover = False
pm = True
color_bird = False

def collision(trubs):
	global pm
	global bird_x
	global bird_y
	global gameover
	for i in range(len(trubs)):
		trub = pygame.Rect(trubs[i][0][0], trubs[i][0][1], 60, 360)
		trub2 = pygame.Rect(trubs[i][1][0], trubs[i][1][1], 60, 360)
		if trub.collidepoint(bird_x + 90, bird_y) or trub2.collidepoint(bird_x + 90, bird_y + 50):
			if pm:
				die.play()
				pm = False
			gameover = True

def update_y(y_change, y_pos):
	global gravity
	y_change += gravity
	y_pos += y_change
	return y_change, y_pos

def update_change(y_change):
	y_change = - y_speed
	return y_change

def update_trubs(trubs):
	global speed
	global score
	if not gameover:
		for i in range(len(trubs)):
			if trubs[i][0][0] < -60:
				y = randint(-370, 0)
				trubs[i][0][1] = y
				trubs[i][1][1] = 600 + y

				trubs[i][0][0] += randint(1100, 1300)
				trubs[i][1][0] = trubs[i][0][0]
				point.play()
				score += 1
			else:
				trubs[i][0][0] -= speed
				trubs[i][1][0] -= speed
	return trubs

while play:
	timer.tick(FPS)

	y_change, bird_y = update_y(y_change, bird_y)
	trubs = update_trubs(trubs)

	screen.blit(bg, (0, 0))
	screen.blit(bird, (bird_x, bird_y))
	game_over_text = pygame.font.Font('Stuff/Samson.ttf', 24).render(('SCORE: ' + str(score)), True, BLUE)
	screen.blit(game_over_text, (0, 0))

	collision(trubs)

	for i in range(len(trubs)):
		pygame.draw.rect(screen, GREEN, trubs[i][0], 0)
		pygame.draw.rect(screen, GREEN, trubs[i][1], 0)

	if gameover:
		game_over_text = pygame.font.Font('Stuff/Samson.ttf', 60).render(('GAME OVER PRESS SPACEBAR'), True, BLUE)
		screen.blit(game_over_text, (170, 260))
	for event in pygame.event.get():
		if event.type == pygame.QUIT:
			play = False

		if event.type == pygame.MOUSEBUTTONDOWN and event.button == 1 and not gameover:
			wing.play()
			y_change = update_change(y_change)

		if event.type == pygame.KEYDOWN:
			if event.key == pygame.K_SPACE and gameover:
				gameover = False
				FPS = 60
				bird_y = 400
				trubs = [[[300, 0, 60, 400], [300, 600, 60 , 770]], [[800, -100, 60, 400], [800, 500, 60 , 770]]]
				y_change = 0
				score = 0
				pm = True
			if event.key == pygame.K_r:
				if color_bird:
					color_bird = False
					bird = pygame.transform.scale(pygame.image.load('Stuff/image/bird.png'), (100,80))
				else:
					color_bird = True
					bird = pygame.transform.scale(pygame.image.load('Stuff/image/wbird.png'), (100,80))
	if bird_y >= 800 or bird_y <= 0:
		if pm:
			die.play()
			pm = False
		gameover = True
	pygame.display.flip()

pygame.quit()


import re
from django.db.models import Q
from django.utils.text import smart_split
from django.views.generic import ListView

from home.models import Noticia
from utils.mixinscomuns import ComunsNoticiasMixin

class NoticiaListView(ComunsNoticiasMixin, ListView):
    model = Noticia
    template_name = "noticias/noticia_list.html"
    paginate_by = 6
    context_object_name = 'list_noticias'
    queryset = Noticia.publicado.select_related()[:50]

class BuscaNoticiaListView(ComunsNoticiasMixin, ListView):
    model = Noticia
    template_name = 'noticias/busca-noticias.html'
    paginate_by = 7
    context_object_name = 'busca_noticias'

    def get_queryset(self, **kwargs):
        query = self.request.GET.get('q', '')
        if query:
            querysets = []
            words = self._extract_terms(query)
            for word in words:
                querysets.append((
                    Q(titulo__icontains=word) |
                    Q(resumo__icontains=word) |
                    Q(secretaria__icontains=word) |
                    Q(conteudo__icontains=word)
                ))

            resultado = Noticia.objects.select_related().filter(*querysets)
        else:
            resultado = []

        return resultado

    def _extract_terms(self, query):
        return [self._clean_term(word) for word in smart_split(query)]

    def _clean_term(self, query):
        return re.sub('\,|\.|\"', '', query)

    def get_context_data(self, **kwargs):
        context = super(BuscaNoticiaListView, self).get_context_data(**kwargs)
        context['search_term'] = self.request.GET['q']
        return context


import numpy as np
from parakeet import jit

@jit
def fdtd(input_grid, steps):
    grid = input_grid.copy()
    old_grid = np.zeros_like(input_grid)
    previous_grid = np.zeros_like(input_grid)

    l_x = grid.shape[0]
    l_y = grid.shape[1]

    for i in range(steps):
        previous_grid[:, :] = old_grid
        old_grid[:, :] = grid
        for x in range(l_x):
            for y in range(l_y):
                grid[x,y] = 0.0
                if 0 < x+1 and x + 1 < l_x:
                    grid[x,y] += old_grid[x+1,y]
                if 0 < x-1 and x- 1 < l_x:
                    grid[x,y] += old_grid[x-1,y]
                if 0 < y+1 and y+1< l_y:
                    grid[x,y] += old_grid[x,y+1]
                if 0 < y-1 and y-1< l_y:
                    grid[x,y] += old_grid[x,y-1]

                grid[x,y] /= 2.0
                grid[x,y] -= previous_grid[x,y]

    return grid

N = 5
steps = 10
input_grid = np.random.randn(N,N).astype('float32')

fdtd(input_grid, steps)


import itertools

n = int(input())
answer = 0

if n < 2:
    print (answer)
elif n == 2:
    print ("1")
else:
    answer = 4
    i = 3
    add = 7
    add2 = 8
    while i != n:
        answer += add
        add += add2
        add2 *= 2
        i += 1
    print (answer)


import numpy as np
import pickle

#import models
paper_data = np.loadtxt('paper_data/published_data.txt')
XGBoost_data = np.loadtxt('XGBoost_data.txt')
DecTrees_data = np.loadtxt('DecisionTree_data.txt')
FFNN_sigmoid_lin_data = np.loadtxt('FFNN_sigmoid_lin_data.txt')
FFNN_sigmoid_tanh_data = np.loadtxt('FFNN_sigmoid_tanh_data.txt')
FFNN_tanh_lin_data = np.loadtxt('FFNN_tanh_lin_data.txt')
FFNN_tanh_tanh_data = np.loadtxt('FFNN_tanh_tanh_data.txt')
Regression_OLS_data = np.loadtxt('Regression_OLS_data.txt')
Regression_ridge_data = np.loadtxt('Regression_ridge_data.txt')
Regression_LASSO_data = np.loadtxt('Regression_LASSO_data.txt')

# Load datasets
with open('datasets.pkl', 'rb') as input:
    Datasets = pickle.load(input)
AME12 = Datasets[0]
AME16 = Datasets[1]
testset = Datasets[2]

#Import experimental values (AME16)
neut = testset.df['N'].to_numpy()
prot = testset.df['Z'].to_numpy()
BAtest = testset.df['B'].to_numpy()
experimental_values = np.concatenate((neut[:,np.newaxis], prot[:,np.newaxis], BAtest[:,np.newaxis]), axis = 1)

#Keep only predicted elements also present in the paper
def keep_only_paper_elements(el_matrix):
    new_matrix = []
    for nucleus1 in paper_data:
        for nucleus2 in el_matrix:
            if (int(nucleus1[0]) == int(nucleus2[0])) and (int(nucleus1[1]) == int(nucleus2[1])):
                new_matrix.append(nucleus2)
                break
    return np.array(new_matrix)

reduced_XGB = keep_only_paper_elements(XGBoost_data)
reduced_DecTrees = keep_only_paper_elements(DecTrees_data)
reduced_FFNN_sigmoid_lin = keep_only_paper_elements(FFNN_sigmoid_lin_data)
reduced_FFNN_sigmoid_tanh = keep_only_paper_elements(FFNN_sigmoid_tanh_data)
reduced_FFNN_tanh_lin = keep_only_paper_elements(FFNN_tanh_lin_data)
reduced_FFNN_tanh_tanh = keep_only_paper_elements(FFNN_tanh_tanh_data)
reduced_regr_OLS = keep_only_paper_elements(Regression_OLS_data)
reduced_regr_ridge = keep_only_paper_elements(Regression_ridge_data)
reduced_regr_LASSO = keep_only_paper_elements(Regression_LASSO_data)

models = [paper_data,
          reduced_XGB,
          reduced_DecTrees,
          reduced_FFNN_sigmoid_lin,
          reduced_FFNN_sigmoid_tanh,
          reduced_FFNN_tanh_lin,
          reduced_FFNN_tanh_tanh,
          reduced_regr_OLS,
          reduced_regr_ridge,
          reduced_regr_LASSO]

model_names = ['paper_data',
              'reduced_XGB',
               'reduced_DecTrees',
               'reduced_FFNN_sigmoid_lin',
               'reduced_FFNN_sigmoid_tanh',
               'reduced_FFNN_tanh_lin',
               'reduced_FFNN_tanh_tanh',
               'reduced_regr_OLS',
               'reduced_regr_ridge',
               'reduced_regr_LASSO']

#compare model to experimental data
def evaluate_model(model):
    if np.array_equal(model, paper_data):
        '''Paper_data contains the difference between the DZ-BNN model
        and the experimental value. We change the model to give the
        same quantity.'''
        diff = paper_data[:,2]
        comparison_list = 0
    else:
        comparison_list = []
        for model_nucleus in model:
            for exp_nucleus in experimental_values:
                if (int(model_nucleus[0]) == int(exp_nucleus[0])) and (int(model_nucleus[1]) == int(exp_nucleus[1])):
                    delta_B = model_nucleus[2] - exp_nucleus[2]
                    comparison_list.append([model_nucleus[0], model_nucleus[1], delta_B])
        comparison_list = np.array(comparison_list)
        diff = comparison_list[:,2]
    return comparison_list, np.std(diff)

comp_lists = []
for i, model in enumerate(models):
    complist, std = evaluate_model(model)
    comp_lists.append(complist)
    print(std, model_names[i])


import tensorflow as tf
#import numpy as np
#import pandas as pd
#import networkx as nx
#import matplotlib
#import matplotlib.pyplot as plt
#from mpl_toolkits.mplot3d import Axes3D
#from pathlib import Path
#import random,math,sympy
#import re
#from turtle import *
#import time,datetime
#import argparse
#import logging
#logger = logging.getLogger()
# logger.setLevel(logging.INFO)
# logging.info('')
# model.summary
# keras.utils.plot_model(model,'model.png')
# keras.utils.plot_model(model,'model_info.png',show_shapes=True)
from tensorflow import keras
from tensorflow.keras import layers

class ThreeLayerMlp(keras.Model):
    def __init__(self, name=None):
        super().__init__(name=name)
        self.dense_1 = layers.Dense(64, activation='relu')
        self.dense_2 = layers.Dense(64, activation='relu')
        self.pred_layer = layers.Dense(10, activation='softmax')

    def call(self, inputs):
        x = self.dense_1(inputs)
        x = self.dense_2(x)
        return self.pred_layer(x)

def get_model():
    return ThreeLayerMlp(name='2_layer_mlp')

model = get_model()
model.summary()
(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()
x_train = x_train.reshape(60000, 784).astype('float32') / 255
x_test = x_test.reshape(10000, 784).astype('float32') / 255
model.compile(loss='sparse_categorical_crossentropy',
        optimizer=keras.optimizers.RMSprop())
history = model.fit(x_train, y_train,batch_size=64,epochs=1)