from __future__ import annotations
from typing import Any, Dict, List, Set, Union
from collections import OrderedDict
from .rows import Rows
from .option import Option
from .columns.column import Column
from .index import Index
from .constraint import Constraint
from ..operations.create_table import CreateTable
class Table:
    _columns: Dict[str, Column] = None
    _constraints: Dict[str, Constraint] = None
    _indexes: Dict[str, Constraint] = None
    _indexed_columns = None
    _name: str = ''
    _options: List[Option] = None
    _primary: Index = None
    _rows: Dict[int, List[Union[str, int]]] = None
    _tracking_rows: bool = False
    _schema_errors: List[str] = None
    _schema_warnings: List[str] = None
    _global_errors: List[str] = None
    _global_warnings: List[str] = None

    def __init__(
        self,
        name: str = '',
        columns: List[Column] = None,
        indexes: List[Index] = None,
        constraints: List[Constraint] = None,
        options: List[Option] = None
    ):
        self._name = name
        self._options = [*options] if options else []
        self._primary = None
        self._constraints = {}
        self._columns = {}
        self._indexes = {}
        self._parsing_errors = None
        self._parsing_warnings = None
        self._schema_errors = None
        self._schema_warnings = None
        self._global_errors = []
        self._global_warnings = []
        self._final_global_errors = None
        self._final_global_warnings = None
        self._raw_columns = columns if columns is not None else []
        self._raw_constraints = constraints if constraints is not None else []
        self._raw_indexes = indexes if indexes is not None else []

        if columns is not None:
            for column in columns:
                self.add_column(column)
        if constraints is not None:
            for constraint in constraints:
                self.add_constraint(constraint)
        if indexes is not None:
            for index in indexes:
                self.add_index(index)

    # Errors are no longer tracked live. Instead, I need to finish adjusting the flow so that if any
    # of the error fetchers are called, then the errors are collected from all children and then the
    # proper errors are returned.

    @property
    def global_errors(self):
        """ Public getter.  Returns a list of schema errors

        :returns: A list of schema errors
        :rtype: list
        """
        if self._schema_errors is None:
            self._find_all_errors()
        return self._final_global_errors

    @property
    def global_warnings(self):
        """ Public getter.  Returns a list of schema warnings

        :returns: A list of schema warnings
        :rtype: list
        """
        if self._schema_errors is None:
            self._find_all_errors()
        return self._final_global_warnings

    @property
    def schema_errors(self):
        """ Public getter.  Returns a list of schema errors

        :returns: A list of schema errors
        :rtype: list
        """
        if self._schema_errors is None:
            self._find_all_errors()
        return self._schema_errors

    @property
    def schema_warnings(self):
        """ Public getter.  Returns a list of schema warnings

        :returns: A list of schema warnings
        :rtype: list
        """
        if self._schema_errors is None:
            self._find_all_errors()
        return self._schema_warnings

    @property
    def parsing_errors(self):
        """ Public getter.  Returns a list of schema errors

        :returns: A list of schema errors
        :rtype: list
        """
        if self._schema_errors is None:
            self._find_all_errors()
        return self._parsing_errors

    @property
    def parsing_warnings(self):
        """ Public getter.  Returns a list of schema warnings

        :returns: A list of schema warnings
        :rtype: list
        """
        if self._schema_errors is None:
            self._find_all_errors()
        return self._parsing_warnings

    @property
    def name(self):
        """ Public getter.  Returns the name of the table.

        :returns: The table name
        :rtype: string
        """

        return self._name

    @property
    def options(self):
        """ Public getter.  Returns a list of table options

        :returns: Table options
        :rtype: list
        """

        return self._options

    def _find_all_errors(self):
        self._schema_errors = []
        self._schema_warnings = []
        self._parsing_errors = []
        self._parsing_warnings = []
        if self._global_errors is not None:
            self._final_global_errors = [*self._global_errors]
        if self._global_warnings is not None:
            self._final_global_warnings = [*self._global_warnings]

        if not self.name:
            self._schema_errors.append("Table missing name")
        if not len(self.columns):
            self._schema_errors.append(f"Table '{self.name}' does not have any columns")

        # start with errors from "children" and append the table name for context
        for type_to_check in [self._columns, self._indexes, self._constraints]:
            for to_check in type_to_check.values():
                for error in to_check.schema_errors:
                    self._schema_errors.append(f"{error} in table '{self.name}'")
                for warning in to_check.schema_warnings:
                    self._schema_warnings.append(f"{warning} in table '{self.name}'")
                for error in to_check.parsing_errors:
                    self._parsing_errors.append(f"{error} in table '{self.name}'")
                for warning in to_check.parsing_warnings:
                    self._parsing_warnings.append(f"{error} in table '{self.name}'")
                if hasattr(to_check, 'global_errors'):
                    for warning in to_check.global_errors:
                        self._final_global_errors.append(f"{error} in table '{self.name}'")
                if hasattr(to_check, 'global_warnings'):
                    for warning in to_check.global_warnings:
                        self._final_global_warnings.append(f"{error} in table '{self.name}'")

        # duplicate names.  This shouldn't really be possible anymore because the add_ methods will
        # throw an exception if we try to add a duplicate, but I'll leave this in just in case.
        for (name, to_check) in {
            'columns': self._columns,
            'constraints': self._constraints,
            'indexes': self._indexes
        }.items():
            if len(to_check) == len(getattr(self, name)):
                continue
            label = name.rstrip('es').rstrip('s')
            found = {}
            duplicates = {}
            for item in to_check:
                if item.name in found:
                    duplicates[item.name] = True
                    continue
                found[item.name] = True
            for key in duplicates.keys():
                self._schema_errors.append(f"Duplicate {label} name found in table '{self.name}': '{key}'")

        # more than one primary key
        primaries = list(filter(lambda index: index.is_primary(), self._indexes.values()))
        if len(primaries) > 1:
            self._schema_errors.append(f"Table '{self.name}' has more than one PRIMARY index")

        # auto increment checks
        if self._columns:
            auto_increment = list(filter(lambda column: column.auto_increment, self._columns.values()))
            if len(auto_increment):
                if len(auto_increment) > 1:
                    self._schema_errors.append(f"Table '{self.name}' has more than one AUTO_INCREMENT column")
                elif not primaries:
                    self._schema_errors.append(
                        f"Table '{self.name}' has an AUTO_INCREMENT column but is missing the PRIMARY index"
                    )
                elif primaries[0].columns[0] != auto_increment[0].name:
                    self.schema_errors.append(
                        "Mismatched indexes in table '%s': column '%s' is the AUTO_INCREMENT column but '%s' is the PRIMARY index column"
                        % (self.name, auto_increment[0].name, primaries[0].columns[0])
                    )

        # indexes on non-existent columns
        for index in self._indexes.values():
            for column in index.columns:
                if not column in self.columns:
                    self._schema_errors.append(
                        f"Table '{self.name}' has index '{index.name}' that references non-existent column '{column}'"
                    )

        # we don't bother checking the constraints to see if they are valid because these are
        # best checked at the database level (since, by definition, foreign key constraints are *typically*
        # against other tables, not within a single table.

    def mark_tracking_rows(self):
        """ Marks the table as having had its rows read, for bookeeping purposes

        The reason this is here is because we keep track of whether or not we are "tracking" rows
        for the current table.  This is for bookeeping purposes, largely as a safe-guard attempt
        to try to identify any more subtle bugs that might creep in.  Normally self._tracking_rows
        gets set to True when we add rows to a table, but if a table is empty then rows will never
        get added, and instead this method must be called to mark the rows as "tracked".
        """
        self._tracking_rows = True
        if self._rows is None:
            self._rows = OrderedDict()

    @property
    def tracking_rows(self) -> bool:
        """ Public getter.  Returns True/False to denote whether or not this table is tracking row records

        To be clear on the distinction: just about any table might have rows.  However,
        that doesn't mean that the mygration system should be syncing rows for that table.
        self.tracking_rows == True denotes that the mygration system thinks that we
        should probably be syncing rows for this table.

        :returns: Whether or not the mygration system is tracking rows on the table
        """
        return self._tracking_rows

    @property
    def columns(self) -> Dict[str, Column]:
        """ Public getter.  Returns an ordered dictionary of table columns """
        return self._columns

    @property
    def indexes(self) -> Dict[str, Index]:
        """ Public getter.  Returns an ordered dictionary of table indexes """
        return self._indexes

    @property
    def constraints(self) -> Dict[str, Constraint]:
        """ Public getter.  Returns an ordered dictionary of table constraints """
        return self._constraints

    @property
    def primary(self) -> Index:
        """ Public getter.  Returns the index object for the primary key """
        return self._primary

    @property
    def rows(self) -> Dict[int, List[Union[str, int]]]:
        """ Public getter.  Returns an ordered dictionary with row data by id """
        return None if self._rows is None else self._rows

    def add_rows(self, rows: Rows) -> Union[str, bool]:
        """ Adds rows to the table

        The rows object has some flexibility in terms of columns: it doesn't just
        assume that a value is provided for every column in the table.  Rather,
        there can be a list of columns and only those columns have values (which
        supports the equivalent of MySQL INSERT queries which only specify values
        for some columns).

        Rows with errors will not be processed

        :returns: An error string if an error is encountered, otherwise True/False
        """
        self._schema_errors = None
        self._schema_warnings = None
        if not isinstance(rows, Rows):
            raise ValueError(
                f"Only objects of class mygrations.core.definitions.rows can be added as rows to a table.  Instead I received an object of class '{rows.__class__.__name__}'"
            )

        if rows.parsing_errors:
            self._global_errors.extend(rows.parsing_errors)
        if rows.parsing_warnings:
            self._global_warnings.extend(rows.parsing_warnings)

        self._tracking_rows = True
        if self._rows is None:
            self._rows = OrderedDict()

        # the rows object may have a list of columns.  If not use our own list of columns
        # remember that self._columns is an OrderedDict so converting its keys to a list
        # actually preserves the order (which is a requirement for us)
        columns = rows.columns if rows.num_explicit_columns else list(self._columns.keys())
        if 'id' not in columns:
            self._global_errors.append(
                "A column named 'id' is required to manage rows in the table, but the id column is missing in the rows for table %s"
                % (self._name, )
            )
            return
        id_index = columns.index('id')

        for values in rows.raw_rows:
            # rows without explicit columns must be checked for matching columns
            if not rows.num_explicit_columns and len(values) != len(columns):
                self._global_errors.append(
                    'Insert values has wrong number of values for table %s and row %s' % (self._name, values)
                )
                continue

            # we need to know the id of this record
            row_id = str(values[id_index])
            if not row_id:
                self._global_errors.append(
                    'Row is missing a value for the id column for table %s and row %s' % (self._name, values)
                )
                continue

            if row_id in self._rows:
                self._global_errors.append('Duplicate row id found for table %s and row %s' % (self.name, values))
                continue

            if not row_id:
                self._global_errors.append('Invalid row id of %s found for table %s' % (row_id, self.name))
                continue

            self._rows[row_id] = OrderedDict(zip(columns, values))
            self._rows[row_id]['id'] = row_id

        return True

    def add_raw_row(self, row: Dict[str, Union[str, int]]) -> Union[str, bool]:
        """ Adds a row into the table as a dictionary instead of a row object

        A bit of repetition here.  This is similar to what happens inside the main
        loop of self.add_rows, but different enough that I'm not trying to combine
        them right this second.

        :returns: An error string if an error is encountered, otherwise True/False
        """
        self._schema_errors = None
        self._schema_warnings = None
        self._tracking_rows = True
        if self._rows is None:
            self._rows = OrderedDict()

        row_id = str(row.get('id'))
        if not row_id:
            raise ValueError("Cannot manage records without an 'id' column and value")
        if row_id in self._rows:
            return 'Duplicate row id found for table %s and row %s' % (self._name, row)

        # make sure we have a value for every column in the row, and build an OrderedDict
        converted_row = OrderedDict()
        for column in self._columns.keys():
            if column in row:
                converted_row[column] = row[column]
            else:
                converted_row[column] = self._columns[column].default
        converted_row['id'] = row_id

        self._rows[row_id] = converted_row

    def column_before(self, column_name: str) -> Union[str, bool]:
        """ Returns the name of the column that comes before a given row.

        Returns true if the column is at the beginning of the table

        :returns: The name of the column before the given column, or True if at the beginning
        """
        # this works because we used an OrderedDict
        columns = [col for col in self.columns.keys()]

        if not column_name in columns:
            raise ValueError(
                "Cannot return column before %s because %s does not exist in table %s" %
                (column_name, column_name, self.name)
            )

        index = columns.index(column_name)
        if index == 0:
            return True

        return columns[index - 1]

    def column_is_indexed(self, column: Union[str, Column]) -> bool:
        """ Returns True/False to denote whether or not the column has a useable index """
        if type(column) != str:
            column = column.name
        if column not in self._columns:
            return False

        # best way to do this is with a set.  We'll keep a record of all indexed columns
        # the column is indexed if an index has that column in the first position
        if self._indexed_columns is None:
            self._indexed_columns = set([index.columns[0] for index in self._indexes.values()])
        return column in self._indexed_columns

    def __str__(self) -> str:
        return str(self.create())

    def create(self, nice=False):
        """ Returns a create table operation that can create this table

        :param nice: Whether or not to return a nicely formatted CREATE TABLE command
        :returns: A create table operation
        """
        return CreateTable(self, nice)

    def nice(self) -> str:
        return str(self.create(True))

    def add_column(self, column: Column, position=False):
        """ Adds a column to the table

        The value of position matches self.position from mygrations.formats.mysql.mygration.operations.add_column

        :param column: The column to add
        :param position: Where to put the column
        """
        self._schema_errors = None
        self._schema_warnings = None
        if column.name in self._columns:
            raise ValueError("Cannot add column %s because %s already exists" % (column.name, column.name))

        # putting it at the end is easy
        if not position:
            self._columns[column.name] = column
            return None

        # beginning is also easy
        if position == True:
            self._columns[column.name] = column
            self._columns.move_to_end(column.name, last=False)
            return None

        # now it is tricky
        found = False
        new_columns = OrderedDict()
        for key, value in self._columns.items():
            new_columns[key] = value
            if key == position:
                new_columns[column.name] = column
                found = True

        if not found:
            raise ValueError(
                "Cannot add column %s after %s because %s does not exist" % (column.name, position, position)
            )

        self._columns = new_columns
        return None

    def remove_column(self, column: Union[str, Column]):
        """ Removes a column from the table """
        self._schema_errors = None
        self._schema_warnings = None
        column_name = column if type(column) == str else column.name
        if not column_name in self._columns:
            raise ValueError("Cannot remove column %s because column %s does not exist" % (column_name, column_name))
        self._columns.pop(column_name, None)

    def change_column(self, new_column: Column):
        """ Changes a column.  This does not currently support renaming columns """
        self._schema_errors = None
        self._schema_warnings = None
        if not new_column.name in self._columns:
            raise ValueError(
                "Cannot modify column %s because column %s does not exist" % (new_column.name, new_column.name)
            )
        self._columns[new_column.name] = new_column

    def add_index(self, index: Index):
        """ Adds an index to the table """
        self._schema_errors = None
        self._schema_warnings = None
        if index.name in self._indexes:
            raise ValueError("Cannot add index %s because index %s already exists" % (index.name, index.name))
        if index.index_type == 'PRIMARY':
            self._primary = index
        self._indexes[index.name] = index

        if self._indexed_columns is not None:
            self._indexed_columns.add(index.columns[0])

    def remove_index(self, index: Union[str, Index]):
        """ Removes an index from the table """
        self._schema_errors = None
        self._schema_warnings = None
        index_name = index if type(index) == str else index.name
        if index_name not in self._indexes:
            raise ValueError("Cannot remove index %s because index %s does not exist" % (index_name, index_name))

        indexed_column = self._indexes[index_name].columns[0]
        self._indexes.pop(index_name, None)
        if self._indexed_columns is not None:
            self._indexed_columns.discard(indexed_column)

    def change_index(self, new_index: Index):
        """ Changes an index.  This does not currently support renaming """
        self._schema_errors = None
        self._schema_warnings = None
        if not new_index.name in self._indexes:
            raise ValueError(
                "Cannot modify index %s because index %s does not exist" % (new_index.name, new_index.name)
            )

        if self._indexed_columns is not None:
            self._indexed_columns.discard(self._indexes[new_index.name].columns[0])
        self._indexes[new_index.name] = new_index

        if self._indexed_columns is not None:
            self._indexed_columns.add(new_index.columns[0])

    def add_constraint(self, constraint: Constraint):
        """ Adds a constraint to the table """
        self._schema_errors = None
        self._schema_warnings = None
        if constraint.name in self._constraints:
            raise ValueError(
                "Cannot add constraint %s because constraint %s already exists" % (constraint.name, constraint.name)
            )
        self._constraints[constraint.name] = constraint

    def remove_constraint(self, constraint: Union[str, Constraint]):
        """ Removes an constraint from the table """
        self._schema_errors = None
        self._schema_warnings = None
        if type(constraint) != str:
            constraint = constraint.name
        if constraint not in self._constraints:
            raise ValueError(
                "Cannot remove constraint %s because constraint %s does not exist" % (constraint, constraint)
            )
        self._constraints.pop(constraint, None)

    def change_constraint(self, new_constraint):
        """ Changes a constraint. This does not currently support renaming. """
        self._schema_errors = None
        self._schema_warnings = None
        if not new_constraint.name in self._constraints:
            raise ValueError(
                "Cannot modify constraint %s because constraint %s does not exist" %
                (new_constraint.name, new_constraint.name)
            )
        self._constraints[new_constraint.name] = new_constraint

    def _loose_equal(self, val1: Union[str, int], val2: Union[str, int]) -> bool:
        """ Performs a looser comparison, as values might have different types depending on whether they came out of a database or file

        Returns true if the two values are equal, even if one is a string and the other an int.
        """
        # if we don't have a type mistmatch then this is easy
        if type(val1) == type(val2):
            return val1 == val2

        # otherwise see if we can cheat and just convert to strings
        return str(val1) == str(val2)

    def to(self, comparison_table, split_operations=False):
        """ Compares two tables to eachother and returns a list of operations which can bring the structure of the second in line with the first

        In other words, this pseudo code will make table have the same structure as comparison_table

        for operation in table.to( comparison_table ):
            table.apply( operation )

        if split_operations is True then a dict of migration operations will be returned to separate
        foreign key operations from everything else.  The dict will have up to three keys: [ 'removed_fks', 'fks', 'kitchen_sink' ]

        'fks' contains the alter statement operation needed to add/change/remove foreign keys
        'kitchen_sink' contains everything else

        If that division of labor seems arbitrary, it isn't: it is separated out that
        way due to the needs of the overall algorithm.

        If split_operations is false then a single alter table operation will be returned that encompasses all changes

        :param comparison_table: A table to find differences with
        :param split_operations: Whether to combine all operations in one alter table or separate them
        :type comparison_table: mygrations.formats.mysql.definitions.table
        :type split_operations: bool
        :returns: A list of operations to apply to table
        :rtype: list[mygrations.formats.mysql.mygrations.operations.*] | dict
        """
        raise NotImplementedError()

    def to_rows(self, from_table=None):
        """ Compares two tables to eachother and returns a list of operations which can bring the rows of this table in line with the other

        It's important to note (and probably important to think through and change eventually)
        that this method has the opposite meaning of `mygrations.formats.mysql.definitions.table.to()`
        That method is called on the `from` table and operations on the (required) `to` table.
        This method is called on the `to` table and can (optionally) be passed in a `from` table.

        :param from_table: A table to find differences with (or None)
        :type from_table: mygrations.formats.mysql.definitions.table
        :returns: A list of operations to apply to table
        :rtype: list[mygrations.formats.mysql.mygrations.operations.*]
        """
        raise NotImplementedError()

    def _differences(self, from_list: Dict[str, Any], to_list: Dict[str, Any]):
        """
        Calculates the difference between two OrderedDicts.

        https://codereview.stackexchange.com/a/176303/140581

        :param from_list: OrderedDict
        :param to_list: OrderedDict
        :return: (added, removed, overlap)
        """

        return ([key for key in to_list if key not in from_list], [key for key in from_list if key not in to_list],
                [key for key in from_list if key in to_list])

    def apply_operation(self, operation):
        """
        Applies an operation to the table

        :param operation: The operation to apply
        :type operation: mygrations.formats.mysql.mygration.operations.*
        """
        operation.apply_to_table(self)


#!/usr/bin/python
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
05.04.Next Number
  Given a positive integer, print the next smallest and the next largest num-
  ber that have the same number of 1 bits in their binary representation.
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
import string
import sys

import ctypes

sys.path.append("../")

from module.ctci  import Exercise
from module.ctci  import Solution

class Ex(Exercise):
  def setup(self):
    self.param.append(8)

class Sol(Solution):
  def csb(self, bits):
    ret = 0
    for b in '{0:b}'.format(bits):
      if b == '1':
        ret += 1
    return ret

  def Naive(self, param):
    v = ctypes.c_uint32(param[0]).value
    csb = self.csb(v)
    V = v+1
    while(1):
      if(csb == self.csb(V)):
        nextLarge = V
        break
      if(V == 2**32 - 1):
        nextLarge = None
        break
      V += 1
    V = v-1
    while(1):
      if(csb == self.csb(V)):
        nextSmall = V
        break
      if(V == 0):
        nextSmall = None
        break
      V -= 1
    return [nextLarge, nextSmall]

  def ON(self, param):
    v = ctypes.c_uint32(param[0]).value

    nextLarget = nextSmall = None
    cnt = 0
    for i in range(32):
      if (((v >> i) & 0b11) ^ 0b10) == 0b11:
        nextLarge = (v ^ (0b01 << i)) | 0b10 << i
        nextLarge = (nextLarge >> i) << i
        nextLarge |= ((1 << i) - 1) >> (i - cnt)
        break
      elif((v >> i) & 0b1):
        cnt += 1

    cnt = 0
    for i in range(32):
      if (((v >> i) & 0b11) ^ 0b01) == 0b11:
        nextSmall = (v ^ (0b10 << i)) | 0b01 << i
        nextSmall = (nextSmall >> i) << i
        nextSmall |= (((1 << i) - 1) >> (i - cnt)) << (i - cnt)
        break
      elif((v >> i) & 0b1):
        cnt += 1

    return [nextLarge, nextSmall]

def solve():
  ex = Ex()

  ex.add_solution(Sol("Naive"))
  ex.add_solution(Sol("ON"))
  ex.solve()

if __name__ == "__main__":
  solve()


import numpy as np
import pandas as pd
from sklearn.preprocessing import MinMaxScaler, LabelEncoder, OneHotEncoder
from scipy import stats
from sklearn.linear_model import LinearRegression, Ridge
from sklearn.preprocessing import PolynomialFeatures
from sklearn.pipeline import make_pipeline
from sklearn.metrics import mean_squared_error
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import cross_val_score
from sklearn.metrics import classification_report
from imblearn.over_sampling import RandomOverSampler

# Data Preprocessing
df_train_clf = pd.read_csv('Classification/train_data.csv')
df_test_clf = pd.read_csv('Classification/test_data.csv')

# Categorical encoding
def one_hot_encode(df: pd.DataFrame, f_name: str):
    ohe = OneHotEncoder(sparse=False).fit_transform(df[f_name].to_numpy().reshape(-1, 1))
    encoded = pd.DataFrame(ohe, columns=[f"{f_name}{i}" for i in range(ohe.shape[1])])
    return pd.concat([df.drop([f_name], axis=1), encoded], axis=1)

f = 'auto_bitrate_state'
ohe_train_clf = one_hot_encode(df_train_clf, f)
ohe_test_clf = one_hot_encode(df_test_clf, f)

def label_encode(df: pd.DataFrame, f_name: str):
    label = LabelEncoder().fit_transform(df[f_name])
    res = df.copy()
    res[f_name] = label
    return res

f = 'auto_fec_state'
prep_train_clf = label_encode(ohe_train_clf, f)
prep_test_clf = label_encode(ohe_test_clf, f)

def scale(df: pd.DataFrame):
    return pd.DataFrame(MinMaxScaler().fit_transform(df), columns=df.columns)

X_train_clf, X_test_clf = scale(prep_train_clf.drop(['stream_quality'], axis=1)), scale(
    prep_test_clf.drop(['stream_quality'], axis=1))
y_train_clf, y_test_clf = prep_train_clf['stream_quality'], prep_test_clf['stream_quality']

# Feature scaling
df_train_reg: pd.DataFrame = pd.read_csv('Regression/train_data.csv')
df_test_reg: pd.DataFrame = pd.read_csv('Regression/test_data.csv')
reg_train_sc = scale(df_train_reg)
reg_test_sc = scale(df_test_reg)
X_train_reg, X_test_reg = reg_train_sc.drop(['target'], axis=1), reg_test_sc.drop(['target'], axis=1)
y_train_reg, y_test_reg = reg_train_sc['target'], reg_test_sc['target']
y_train_clf, y_test_clf = y_train_clf.astype(int), y_test_clf.astype(int)

# Dimensionality reduction (exact reasoning why I did it this way can be seen in ipynb)
X_train_reg, X_test_reg = X_train_reg.drop(['rtt_std'], axis=1), X_test_reg.drop(['rtt_std'], axis=1)
X_train_clf, X_test_clf = X_train_clf.drop(['auto_bitrate_state0'], axis=1), X_test_clf.drop(['auto_bitrate_state0'],
                                                                                axis=1)

# Data visualization can be seen in the ipynb

# Regression model
model1 = LinearRegression()
model2 = Ridge()
model1.fit(X_train_reg, y_train_reg)
model2.fit(X_train_reg, y_train_reg)

y_pred_reg1 = model1.predict(X_test_reg)
y_pred_reg2 = model2.predict(X_test_reg)

# Finding best polynomial regression model
res = []

for i in range(2, 5):
    poly = make_pipeline(PolynomialFeatures(i), LinearRegression())
    poly.fit(X_train_reg, y_train_reg)
    y_pred_reg3 = poly.predict(X_test_reg)
    res.append(mean_squared_error(y_test_reg, y_pred_reg3))
print('----------------------------------------------------')
print(f'Results of Polynomial Regression: {res}')

# Logistic Regression model

logistic = LogisticRegression(penalty='l2', max_iter=120000)
logistic.fit(X_train_clf, y_train_clf)
y_pred_clf = logistic.predict(X_test_clf)

# Results evaluation
# Regression

model1 = LinearRegression()
model2 = Ridge()
model3 = make_pipeline(PolynomialFeatures(2), LinearRegression())

m1_cross = cross_val_score(model1, X_train_reg, y_train_reg, cv=5)
m2_cross = cross_val_score(model2, X_train_reg, y_train_reg, cv=5)
m3_cross = cross_val_score(model3, X_train_reg, y_train_reg, cv=5)

print('----------------------------------------------------')

print(f'Model 1 Cross Validation mean: {np.mean(m1_cross)}, std: {np.std(m1_cross)}')
print(f'Model 2 Cross Validation mean: {np.mean(m2_cross)}, std: {np.std(m2_cross)}')
print(f'Model 3 Cross Validation mean: {np.mean(m3_cross)}, std: {np.std(m3_cross)}')
print('----------------------------------------------------')
print(f'Model 1 (Linear) - MSE:{mean_squared_error(y_test_reg, y_pred_reg1)}')
print(f'Model 2 (Linear + Ridge) - MSE:{mean_squared_error(y_test_reg, y_pred_reg2)}')
print(f'Model 3 (Polynomial with degree {np.argmin(res) + 2}) - MSE:{res[np.argmin(res)]}')

# Classification

logistic = LogisticRegression(penalty='l2', max_iter=120000)

log_cross = cross_val_score(logistic, X_train_clf, y_train_clf, cv=5)
print('----------------------------------------------------')
print(f'Logistic Regression Cross Validation mean: {np.mean(log_cross)}, std: {np.std(log_cross)}')
print('----------------------------------------------------')
print(classification_report(y_test_clf, y_pred_clf))

# Outliers detection
# Finding outliers
X_train_clf_o = X_train_clf.copy()
X_train_clf_o['target'] = y_train_clf
z = np.abs(stats.zscore(X_train_clf))

X_train_clf_o = X_train_clf_o[(z < 3).all(axis=1)]
print('----------------------------------------------------')
print(f'Removed Outliers: {X_train_clf.shape[0] - X_train_clf_o.shape[0]}')

X_train_clf_o, y_train_clf_o = X_train_clf_o.drop('target', axis=1), X_train_clf_o['target']

smt = RandomOverSampler(sampling_strategy='minority')
X_res, y_res = smt.fit_resample(X_train_clf_o, y_train_clf_o)

logistic = LogisticRegression(penalty='l2', max_iter=120000)

logistic.fit(X_res, y_res)
print('----------------------------------------------------')
print(f'Number of classes: 0 - {sum(y_res == 0)}, 1 - {sum(y_res == 1)}')

log_cross = cross_val_score(logistic, X_res, y_res, cv=5)
print('----------------------------------------------------')
print(f'Logistic Regression with sampling Cross Validation mean: {np.mean(log_cross)}, std: {np.std(log_cross)}')
print('----------------------------------------------------')
y_pred = logistic.predict(X_test_clf)
print('After outliers detection')
print(classification_report(y_test_clf, y_pred))
print('Before outliers detection')
print(classification_report(y_test_clf, y_pred_clf))


# -*- coding: utf-8 -*-
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

from odoo import models, fields, api

from of_datastore_product import DATASTORE_IND

# Création/édition d'objets incluant un article centralisé
class OfDatastoreProductReference(models.AbstractModel):
    _name = 'of.datastore.product.reference'

    @api.model
    def create(self, vals):
        # par défaut .get() retourne None si la clef n'existe pas, et None == -1
        # Gestion des many2one
        if vals.get('product_id', 0) < 0:
            vals['product_id'] = self.env['product.product'].browse(vals['product_id']).of_datastore_import().id

        # Gestion des many2many
        if vals.get('product_ids') and vals['product_ids'][0][2]:
            res = []
            product_ids = vals['product_ids'][0][2]
            ds_products = self.env['product.product'].browse([pid for pid in product_ids if pid < 0])

            # On appelle of_datastore_import() ici plutôt que dans le for
            # pour éviter d'appeler trop souvent la base centralisée
            products = ds_products.of_datastore_import()
            ds_products_dict = {p.of_datastore_res_id: p.id for p in products}

            for pid in product_ids:
                if pid < 0:
                    pid = ds_products_dict[-pid % DATASTORE_IND]
                res.append(pid)
            vals['product_ids'] = [[6, 0, res]]

        return super(OfDatastoreProductReference, self).create(vals)

    @api.multi
    def write(self, vals):
        # Gestion des many2one
        if vals.get('product_id', 0) < 0:
            vals['product_id'] = self.env['product.product'].browse(vals['product_id']).of_datastore_import().id

        # Gestion des many2many
        if vals.get('product_ids') and vals['product_ids'][0][2]:
            res = []
            product_ids = vals['product_ids'][0][2]
            ds_products = self.env['product.product'].browse([pid for pid in product_ids if pid < 0])

            # On appelle of_datastore_import() ici plutôt que dans le for
            # pour éviter d'appeler trop souvent la base centralisée
            products = ds_products.of_datastore_import()
            ds_products_dict = {p.of_datastore_res_id: p.id for p in products}

            for pid in product_ids:
                if pid < 0:
                    pid = ds_products_dict[-pid % DATASTORE_IND]
                res.append(pid)
            vals['product_ids'] = [[6, 0, res]]

        return super(OfDatastoreProductReference, self).write(vals)

class OfProductKitLine(models.Model):
    _name = 'of.product.kit.line'
    _inherit = ['of.product.kit.line', 'of.datastore.product.reference']

    @api.multi
    def _of_read_datastore(self, fields_to_read, create_mode=False):
        u"""
        Lit les données des kits dans leur base fournisseur.
        @param ids: id modifié des produits, en valeur négative
        """
        supplier_obj = self.env['of.datastore.supplier']
        brand_obj = self.env['of.product.brand']
        product_obj = self.env['product.product']
        res = []

        # Kits par fournisseur
        datastore_kit_ids = {}
        for full_id in self._ids:
            supplier_id = -full_id / DATASTORE_IND
            datastore_kit_ids.setdefault(supplier_id, []).append((-full_id) % DATASTORE_IND)

        for supplier in supplier_obj.browse(datastore_kit_ids):
            client = supplier.of_datastore_connect()
            ds_kit_obj = supplier.of_datastore_get_model(client, 'of.product.kit.line')

            # Données de la base fournisseur
            kits_data = supplier.of_datastore_read(ds_kit_obj, datastore_kit_ids[supplier.id], [])
            ds_product_ids = [kit['product_id'][0] for kit in kits_data]

            # Détection des composants du kit déjà importés
            # Attention de bien détecter les articles archivés (pourrait sinon provoquer des erreurs lors de l'import)
            products = product_obj.with_context(active_test=False).search(
                [('brand_id', 'in', supplier.brand_ids.ids),
                 ('of_datastore_res_id', 'in', ds_product_ids)])

            product_match = {product.of_datastore_res_id: product.id for product in products}
            product_names = dict(products.name_get())

            # Affectation des ids des champs relationnels
            supplier_value = supplier.id * DATASTORE_IND
            match_dicts = {}
            for kit in kits_data:
                # Articles
                product_id, product_name = kit['product_id']
                if product_id in product_match:
                    # Composant déjà importé
                    product_id = product_match[product_id]
                    product_name = product_names[product_id]
                else:
                    # Composant virtuel
                    product_id = -(product_id + supplier_value)
                kit['product_id'] = (product_id, product_name)
                kit['id'] = -(kit['id'] + supplier_value)

                # Unités de mesure
                uom_id, uom_name = kit['product_uom_id']
                uom_id = brand_obj.datastore_match(
                    client, 'product.uom', uom_id, uom_name, False, match_dicts, create=create_mode).id
                kit['product_uom_id'] = (uom_id, uom_name)
            res += kits_data
        return res

    @api.multi
    def read(self, fields=None, load='_classic_read'):
        lines = self.filtered(lambda o: o.id > 0)
        ds_lines = self - lines

        # kits sur la base courante
        res = super(OfProductKitLine, self).read(fields, load=load)

        if ds_lines:
            res += ds_lines._of_read_datastore(fields)
        return res

class SaleOrderLine(models.Model):
    _name = 'sale.order.line'
    _inherit = ['sale.order.line', 'of.datastore.product.reference']

    of_brand_id = fields.Many2one(
        'of.product.brand', string="Filtre de marque",
        help=u"Ce champ permet de restreindre les articles proposés à une seule marque.\n"
             u"Renseigner ce champ permet aussi la recherche dans le tarif centralisé.\n"
             u"Utilisation équivalente au code m:XXX dans la recherche d'article, où XXX est le code de la marque")

    @api.multi
    @api.onchange('product_id')
    def product_id_change(self):
        if self.product_id and self.of_brand_id and self.product_id.brand_id != self.of_brand_id:
            self.of_brand_id = False
        return super(SaleOrderLine, self).product_id_change()

    @api.multi
    @api.onchange('of_brand_id')
    def _onchange_of_brand_id(self):
        if self.product_id and self.of_brand_id and self.product_id.brand_id != self.of_brand_id:
            self.product_id = False

class SaleOrderLineComp(models.Model):
    _name = 'of.saleorder.kit.line'
    _inherit = ['of.saleorder.kit.line', 'of.datastore.product.reference']

class AccountInvoiceLine(models.Model):
    _name = 'account.invoice.line'
    _inherit = ['account.invoice.line', 'of.datastore.product.reference']

class AccountInvoiceLineComp(models.Model):
    _name = 'of.invoice.kit.line'
    _inherit = ['of.invoice.kit.line', 'of.datastore.product.reference']

class PurchaseOrderLine(models.Model):
    _name = 'purchase.order.line'
    _inherit = ['purchase.order.line', 'of.datastore.product.reference']

class StockInventoryLine(models.Model):
    _name = 'stock.inventory.line'
    _inherit = ['stock.inventory.line', 'of.datastore.product.reference']

class OFParcInstalle(models.Model):
    _name = 'of.parc.installe'
    _inherit = ['of.parc.installe', 'of.datastore.product.reference']


import matplotlib.pyplot as plt
import compiled

def draw():
    x_points = []
    y_points = []

    for x in range(-1000, 1000):
        try:
            y_points.append(compiled.compiled_func(x))
            x_points.append(x)
        except:
            print("Error, posible discontinuidad en: ", x)
    plt.scatter(x_points, y_points)
    plt.show()


import unittest
import sqlite3
import json
import os
import matplotlib.pyplot as plt
import requests
import plotly.graph_objects as go
import plotly.express as px
import pandas as pd
import csv
# starter code

def setUpDatabase(db_name):
    path = os.path.dirname(os.path.abspath(__file__))
    conn = sqlite3.connect(path+'/'+db_name)
    cur = conn.cursor()
    return cur, conn

def data_calc(cur, conn):
    data = {}
    lst = []
    cur.execute('SELECT Country_IDs.country, Covid_Cases.cases_feb_01_2020, Covid_Cases.cases_feb_01_2021, CO2_Emissions.emissions_2020, CO2_Emissions.emissions_2021 FROM Country_IDs JOIN Covid_Cases ON Country_IDs.country_id = Covid_Cases.country_id JOIN CO2_Emissions ON CO2_Emissions.country_id = Country_IDs.country_id')

    lst = cur.fetchall()
    for item in lst:
        data[item[0]] = {}
        data[item[0]]['cases_difference'] = round(((item[2] - item[1]) / 100000), 2)
        data[item[0]]['emissions_difference'] = (round((item[4] - item[3]), 2))

    return data

def emissions_calc(cur, conn):
    data = {}
    lst = []
    cur.execute('SELECT Country_IDs.country, CO2_Emissions.emissions_2020, CO2_Emissions.emissions_2021 FROM Country_IDs JOIN CO2_Emissions ON CO2_Emissions.country_id = Country_IDs.country_id')
    lst = cur.fetchall()
    total_change = 0

    for item in lst:
        data[item[0]] = {}
        data[item[0]]['difference'] = round((item[2] - item[1]), 2)

    return data

def cases_calc(cur, conn):
    data = {}
    lst = []
    cur.execute('SELECT Country_IDs.country, Covid_Cases.cases_feb_01_2020, Covid_Cases.cases_feb_01_2021 FROM Country_IDs JOIN Covid_Cases ON Covid_Cases.country_id = Country_IDs.country_id')
    lst = cur.fetchall()
    total_change = 0

    for item in lst:
        data[item[0]] = {}
        data[item[0]]['difference'] = round((item[2] - item[1]) , 2)

    return data

def scatter_plot_vis(data):
    country_lst = []
    emissions_lst = []
    covid_cases_lst = []

    for item in data:
        country_lst.append(item)
    country_lst = country_lst[0:10]

    for country in country_lst:
        covid_cases_lst.append(data[country]['cases_difference'])

    for country in country_lst:
        emissions_lst.append(data[country]['emissions_difference'])

    scatter_data = {
        'Change in Covid Cases (# of cases)' : covid_cases_lst,
        'Change in CO2 Emissions (mtons)' : emissions_lst,
        'Country' : country_lst,
        'Size' : 5,
        }
    df = pd.DataFrame(scatter_data)

    fig = px.scatter(
        df, x= 'Change in CO2 Emissions (mtons)',
        y = 'Change in Covid Cases (# of cases)',
        title = "February 2020 - February 2021",
        hover_data = ['Country'],
        size = 'Size'
        )

    fig.update_layout(
        font_family = 'Courier New',
        title_font_family = 'Times New Roman'
    )

    fig.show()

def co2_covid_vis(data):
    country_lst = []
    emissions_lst = []
    covid_cases_lst = []

    for item in data:
        country_lst.append(item)

    # get the top emissions for 2021
    country_lst = country_lst[0:10]

    for country in country_lst:
        covid_cases_lst.append(data[country]['cases_difference'])
        emissions_lst.append(data[country]['emissions_difference'])

    fig = go.Figure(data = [
        go.Bar(name = "Covid Cases Change (in 100,000 cases)", x = country_lst, y = covid_cases_lst, marker_color = 'rgb(255, 127, 80) '),
        go.Bar(name = "Emissions Change (in Mtons)", x = country_lst, y = emissions_lst, marker_color = 'rgb(100, 149, 237)')])
    fig.update_layout(barmode='group')
    fig.show()

def pie_chart(data, title):
    countries_lst = []
    percent_lst = []
    for item in data:
        countries_lst.append(item)

    countries_lst = countries_lst[0:10]

    for country in countries_lst:
        percent_lst.append(data[country]['difference'])

    fig = px.pie(values=percent_lst, names=countries_lst, title=title)
    fig.show()

def main():
    cur, conn = setUpDatabase('api_data.db')
    co2_covid_data = data_calc(cur, conn)
    co2_covid_vis(co2_covid_data)
    scatter_plot_vis(co2_covid_data)

    emissions_p_data = emissions_calc(cur, conn)
    pie_chart(emissions_p_data, 'CO2 Emissions Change by Country (as a percent of total change) Feb 2020 - Feb 2021')

    cases_p_data = cases_calc(cur,conn)
    pie_chart(cases_p_data, 'Total Covid Cases Change (as a percent of total change) by Country Feb 2020 - Feb 2021')

    small_lst = []
    all_lst = []

    for i in co2_covid_data:
        small_lst.append(i)
        small_lst.append(co2_covid_data[i]['cases_difference'])
        small_lst.append(co2_covid_data[i]['emissions_difference'])
        all_lst.append(small_lst)

    column_names = ['country', 'cases_difference', 'emissions_difference']

    with open('calculations.csv', 'w') as csvfile:
        write = csv.writer(csvfile)
        write.writerow(column_names)

        for i in co2_covid_data:
            small_lst = []
            small_lst.append(i)
            small_lst.append(co2_covid_data[i]['cases_difference'])
            small_lst.append(co2_covid_data[i]['emissions_difference'])
            write.writerow(small_lst)

    conn.close()

if __name__ == "__main__":
    main()
    unittest.main(verbosity=2)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Setup script
"""

from setuptools import setup
import os

def read(fname):
    """returns the text of a file"""
    return open(os.path.join(os.path.dirname(__file__), fname), 'r').read()

def get_requirements(filename="requirements.txt"):
    """returns a list of all requirements"""
    text = read(filename)
    requirements = []
    for line in text.splitlines():
        req = line.split('#')[0].strip()
        if req != '':
            requirements.append(req)
    return requirements

setup(
    name='dockgraph',
    version='0.1',
    # see https://pypi.python.org/pypi?%3Aaction=list_classifiers
    classifiers=[
        'Development Status :: 1 - Planning',
        'Environment :: Console',
        'Intended Audience :: Developers',
        'Intended Audience :: System Administrators',
        'License :: Other/Proprietary License',
        'Natural Language :: English',
        'Operating System :: OS Independent',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3',
        'Topic :: Utilities'
    ],
    description='analyse dependencies of docker images',
    long_description=read('README.rst'),
    url='https://github.com/jneureuther/dockgraph',
    author='Julian Neureuther',
    author_email='dev@jneureuther.de',
    license=read('LICENSE'),
    packages=['dockgraph'],
    scripts=['bin/dockgraph'],
    test_suite="tests",
    install_requires=get_requirements(),
)


import logging
import sqlite3
from sqlite3 import Error

logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.DEBUG)

class RecordsHandler:
    def __init__(self, db_file):
        self.sql_create_projects_table = """ CREATE TABLE IF NOT EXISTS Records (
                                        id integer PRIMARY KEY,
                                        name text NOT NULL,
                                        email text NOT NULL,
                                        age text NOT NULL,
                                        origin text NOT NULL
                                    ); """

        self.create_connection(db_file)

    def create_connection(self, db_file):
        """ create a database connection to a SQLite database """
        self.conn = None

        self.conn = sqlite3.connect(db_file)
        logging.info('Connection established to ' + db_file)

        self.create_table(self.sql_create_projects_table)

    def close_connection(self):
        if self.conn:
            self.conn.close()
            logging.info('Connection closed')

    def create_table(self, create_table_sql):
        """create a table from the create_table_sql statement
        :param conn: Connection object
        :param create_table_sql: a CREATE TABLE statement
        :return:
        """
        c = self.conn.cursor()
        c.execute(create_table_sql)
        self.conn.commit()
        logging.info('Creating Records table')

    def add_record(self, name="", email="", age="", origin=""):
        sqlquery = "INSERT INTO Records (name, email, age, origin) VALUES ('%s', '%s', '%s', '%s')"
        val = (name, email, age, origin)
        query = sqlquery % val

        c = self.conn.cursor()
        c.execute(query)
        self.conn.commit()
        logging.info('New record added')

    def delete_record(self, record_id=-1):
        sqlquery = "DELETE FROM Records WHERE id='%s'"
        val = record_id
        query = sqlquery % val

        c = self.conn.cursor()
        c.execute(query)
        self.conn.commit()
        logging.info('Record with id ' + str(record_id) + ' deleted.')

    def look(self, email="", age=""):
        sqlquery = "SELECT * FROM Records WHERE email='%s' AND age='%s'"
        val = (email, age)
        query = sqlquery % val

        results = []

        c = self.conn.cursor()
        c.execute(query)
        results.extend(c.fetchall())

        return results

    def list_all(self):
        query = "SELECT * FROM Records"
        results = []

        c = self.conn.cursor()
        c.execute(query)
        results.extend(c.fetchall())

        # loop through the rows
        for row in results:
            print(row)

    def delete_all(self):
        sqlquery = "DELETE FROM Records"

        c = self.conn.cursor()
        c.execute(sqlquery)
        self.conn.commit()


class malDataExport:
    # data can be the map from anilist.trimList
    def __init__(self, data):
        self.entries = []
        for x in data:
            self.entries.append(malEntry(x))

    def mal_skeleton(self, name):
        # chr(10) == '\n'
        return f"""<?xml version="1.0" encoding="UTF-8" ?>
		<myanimelist>
			<myinfo>
				<user_name>{name}</user_name>
				<user_export_type>1</user_export_type>
			</myinfo>
                            {chr(10).join(map(str,self.entries))}
			</myanimelist>
        """

class malEntry:
    def __init__(self, entry):
        self.id = entry['media']['idMal']
        self.progress = entry['progress']
        self.date_start = entry['startedAt']
        self.date_end = entry['completedAt']
        self.score = entry['score']
        self.status = entry['status']
        self.repeat_number = entry['repeat']
        self.comment = entry['notes']
        self.convert_status()
        self.convert_dates()

    def convert_status(self):
        switch = {
            'CURRENT': 'Watching',
            'PLANNING': 'Plan to Watch',
            'COMPLETED': 'Completed',
            'DROPPED': 'Dropped',
            'PAUSED': 'On-Hold',
            'REPEATING': 'Completed',
        }
        self.status = switch.get(self.status)

    def convert_dates(self):
        self.date_start = malEntry.convert_date(self.date_start)
        self.date_end = malEntry.convert_date(self.date_end)

    @staticmethod
    def convert_date(date):
        if date['year'] == None:
            return '0000-00-00'
        else:
            return '{}-{}-{}'.format(date['year'], date['month'], date['day'])

    def __str__(self):
        return f"""<anime>
    <series_animedb_id>{self.id}</series_animedb_id>
    <my_watched_episodes>{self.progress}</my_watched_episodes>
    <my_start_date>{self.date_start}</my_start_date>
    <my_finish_date>{self.date_end}</my_finish_date>
    <my_score>{self.score}</my_score>
    <my_status>{self.status}</my_status>
    <my_times_watched>{self.repeat_number}</my_times_watched>
    <my_tags><![CDATA[{self.comment}]]></my_tags>
    <update_on_import>1</update_on_import>
</anime>"""

    __repr__ = __str__


import pandas as pd
import pickle

def clean(list_):
    list_ = list_.replace(' ','')
    list_ = list_.replace(',', ' ')
    list_ = list_.replace("'", '')
    list_ = list_.replace('[', '')
    list_ = list_.replace(']', '')
    return list_

# rate all movies based in the true bayesian estimation (refer README.md)
def rating(df):

    m = df.numVotes.quantile(0.80)
    C = df.averageRating.mean()

    df['weightedRating'] = ((df['numVotes'] / (df['numVotes'] + m) )*df['averageRating']) + ((m/ (m + df['numVotes']))*C )
    df.sort_values('weightedRating', ascending=False, inplace=True)

    return df

# store all possible genres in a pickle file
def genres(df):
    # drop all null genres
    df.drop(df[df['genres'] == '\\N'].index, inplace = True)

    # filter out only valid years
    df = df[df.startYear.str.isnumeric()]

    # convert string of genres to list
    df['genres'] = df['genres'].str.split(',')

    genres = df.genres.explode().unique()

    # pickle file of possible genres
    with open('../data/genres.pkl', 'wb') as f:
        pickle.dump(genres, f)
    return df

# clean the data so they can be combined to form a final string of parameters, remove space between first name and last name
def clean_data(df):
    df['nconst'] = df['nconst'].fillna('[]')
    df['nconst'] = df['nconst'].apply(clean)

    df['directors'] = df['directors'].fillna('')
    df['directors'] = df['directors'].apply(clean)

    df['writers'] = df['writers'].fillna('')
    df['writers'] = df['writers'].apply(clean)

    return df

# combine all parameters together
def soup(df):
    # make a string out of list of genre
    df['genreString'] = df['genres'].str.join(' ')

    #director is included three times in the soup as it should have more weight as compared to the other cast
    df['soup'] = df['keywords'] + ' ' + df['nconst'] + ' ' +  df['genreString'] + ' ' +  df['writers'] + ' ' +  df['directors'] + ' ' +  df['directors'] +  ' ' + df['directors'] + ' ' + df['overview']
    df = df.drop(['genreString'], axis = 1)
    df['soup'] = df['soup'].fillna('')

    return df

df = pd.read_csv('../data/data.csv')

df = genres(df)
df = rating(df)
df = clean_data(df)
df = soup(df)

df = df[(df['originalLanguage'] == 'hi') | (df['originalLanguage'] == 'en')]
df['poster'] = df['poster'].fillna(' ')
df = df[['titleId','displayTitle','genres', 'startYear','originalLanguage', 'soup', 'weightedRating', 'overview', 'poster', 'directors','averageRating','nconst']]

df.to_pickle('../data/data.pkl')
df.to_csv('../data/rated_data.csv')


import torch, torchvision
from torch import nn

img_hidden_sz = 512
num_image_embeds = 5

#?? 8907
n_classes = 8790
img_embed_pool_type = 'avg'

class ImageEncoder18(nn.Module):
    def __init__(self):
        super(ImageEncoder18, self).__init__()
        model = torchvision.models.resnet18(pretrained=True)
        modules = list(model.children())[:-2]
        self.model = nn.Sequential(*modules)

        pool_func = (
            nn.AdaptiveAvgPool2d
            if img_embed_pool_type == "avg"
            else nn.AdaptiveMaxPool2d
        )

        if num_image_embeds in [1, 2, 3, 5, 7]:
            self.pool = pool_func((num_image_embeds, 1))
        elif num_image_embeds == 4:
            self.pool = pool_func((2, 2))
        elif num_image_embeds == 6:
            self.pool = pool_func((3, 2))
        elif num_image_embeds == 8:
            self.pool = pool_func((4, 2))
        elif num_image_embeds == 9:
            self.pool = pool_func((3, 3))

    def forward(self, x):
        # Bx3x224x224 -> Bx512x7x7 -> Bx512xN -> BxNx512
        out = self.pool(self.model(x))
        out = torch.flatten(out, start_dim=2)
        out = out.transpose(1, 2).contiguous()
        return out  # BxNx2048

class ImageClf18(nn.Module):
    def __init__(self):
        super(ImageClf18, self).__init__()
        self.img_encoder = ImageEncoder18()
        self.clf = nn.Linear(img_hidden_sz * num_image_embeds, n_classes)

    def forward(self, x):
        x = self.img_encoder(x)
        x = torch.flatten(x, start_dim=1)
        out = self.clf(x)
        return out


import os
import sys
sys.path.append('..')
sys.path.append('../..')
import argparse
import utils
from tsp_helper import *
from student_utils import *

OUTPUT_FILENAME = "naive_output.txt"

"""
======================================================================
  Complete the following function.
======================================================================
"""

def solve(list_of_locations, list_of_homes, starting_car_location, adjacency_matrix, params=[]):
    """
    Write your algorithm here.
    Input:
        list_of_locations: A list of locations such that node i of the graph corresponds to name at index i of the list
        list_of_homes: A list of homes
        starting_car_location: The name of the starting location for the car
        adjacency_matrix: The adjacency matrix from the input file
    Output:
        A list of locations representing the car path
        A dictionary mapping drop-off location to a list of homes of TAs that got off at that particular location
        NOTE: both outputs should be in terms of indices not the names of the locations themselves
    """
    # Ensure that the starting location is at the start.
    homes_and_start = list_of_homes.copy()
    if starting_car_location not in list_of_homes:
        homes_and_start.insert(0, starting_car_location)
    else:
        homes_and_start.remove(starting_car_location)
        homes_and_start.insert(0, starting_car_location)

    # Initialize storage structs (see below)
    sp_lookup = {}
    only_tas_adjacency = []

    # Get shortest paths between all houses w. networkx
    G, error = adjacency_matrix_to_graph(adjacency_matrix)
    for i, home1 in enumerate(homes_and_start):
        home1_index = list_of_locations.index(home1)
        home1_row = []
        for j, home2 in enumerate(homes_and_start):
            if home1 == home2:
                home1_row.append(0)
                continue
            home2_index = list_of_locations.index(home2)
            sp = nx.shortest_path(G, source=home1_index, target=home2_index)
            cost = cost_of_path(sp, adjacency_matrix)
            # Construct lookup dict ((i, j), (A, ... , B))
            # i.e. ((int, int), (int, ..., int))
            sp_lookup[(i, j)] = (tuple(sp))

            home1_row.append(cost)
        # Create adjacency mtx of only TA homes and start, for use in ortools tsp solver
        only_tas_adjacency.append(home1_row)

    # Plus into tsp_solver, get back shortest tour of TA houses
    shortest_tour = get_tsp_result(only_tas_adjacency, 0)

    # Replace house paths w actual paths from dict
    car_tour = []
    is_start = True
    first_ind = 0
    for index in range(len(shortest_tour) - 1):
        i, j = shortest_tour[index], shortest_tour[index + 1]
        sp = sp_lookup[(i, j)]
        if is_start:
            first_ind = sp[0]
            is_start = False
        sp_without_last = sp[:len(sp_lookup[(i, j)]) - 1]
        for elem in sp_without_last:
            car_tour.append(elem)
    car_tour.append(first_ind)

    # Each TA gets dropped off at their house.
    dropoff_dict = {}
    for location_ind in car_tour:
        if list_of_locations[location_ind] in list_of_homes:
            dropoff_dict[location_ind] = [location_ind]

    # Done
    print(car_tour)
    print(dropoff_dict)
    return car_tour, dropoff_dict

""" Lookup in adj matrix the cost of some path"""
def cost_of_path(path, adj_matrix):
    total, start, next = 0, 0, 1
    while next < len(path):
        total += adj_matrix[path[start]][path[next]]
        start += 1
        next += 1
    return total

"""
======================================================================
   No need to change any code below this line
======================================================================
"""

"""
Convert solution with path and dropoff_mapping in terms of indices
and write solution output in terms of names to path_to_file + file_number + '.out'
"""
def convertToFile(path, dropoff_mapping, path_to_file, list_locs):
    string = ''
    for node in path:
        string += list_locs[node] + ' '
    string = string.strip()
    string += '\n'

    dropoffNumber = len(dropoff_mapping.keys())
    string += str(dropoffNumber) + '\n'
    for dropoff in dropoff_mapping.keys():
        strDrop = list_locs[dropoff] + ' '
        for node in dropoff_mapping[dropoff]:
            strDrop += list_locs[node] + ' '
        strDrop = strDrop.strip()
        strDrop += '\n'
        string += strDrop
    utils.write_to_file(path_to_file, string)

def solve_from_file(input_file, output_directory, params=[]):
    print('Processing', input_file)

    input_data = utils.read_file(input_file)
    num_of_locations, num_houses, list_locations, list_houses, starting_car_location, adjacency_matrix = data_parser(input_data)
    car_path, drop_offs = solve(list_locations, list_houses, starting_car_location, adjacency_matrix, params=params)

    basename, filename = os.path.split(input_file)
    if not os.path.exists(output_directory):
        os.makedirs(output_directory)
    output_file = utils.input_to_output(input_file, output_directory)

    convertToFile(car_path, drop_offs, output_file, list_locations)

def solve_all(input_directory, output_directory, params=[]):
    input_files = utils.get_files_with_extension(input_directory, 'in')

    for input_file in input_files:
        solve_from_file(input_file, output_directory, params=params)

if __name__=="__main__":
    parser = argparse.ArgumentParser(description='Parsing arguments')
    parser.add_argument('--all', action='store_true', help='If specified, the solver is run on all files in the input directory. Else, it is run on just the given input file')
    parser.add_argument('input', type=str, help='The path to the input file or directory')
    parser.add_argument('output_directory', type=str, nargs='?', default='.', help='The path to the directory where the output should be written')
    parser.add_argument('params', nargs=argparse.REMAINDER, help='Extra arguments passed in')
    args = parser.parse_args()
    output_directory = args.output_directory
    if args.all:
        input_directory = args.input
        solve_all(input_directory, output_directory, params=args.params)
    else:
        input_file = args.input
        solve_from_file(input_file, output_directory, params=args.params)


# -*- coding: utf-8 -*-
"""
Created on Thu Mar 31 16:17:59 2016

Consider Rotation and Glide vector both

@author: Neo
"""

#
#def Rotation_Glide(ua,ud,sigua,sigud,alp0,det0):
#        return [w,sigw,wx,wy,wz,sigwx,sigwy,sigwz,\
#            g,sigg,gx,gy,gz,siggx,siggy,siggz,chi]

from fun import Rotation_Glide
from Data_Load import Rank_Load

res_dir = '../results/'
#dat_fil = 'NC3.rank'
#res = 'Rank_RG.dat'

#dat_fil = 'GR.rank'
#res = 'GR2.dat'
dat_fil = 'OV.rank'
res = 'OV2.dat'
fres = open(res_dir + res,'w')

[Sou, V, V_err, V_RA,V_Dec, Va_E, Vd_E, RA, Dec] = Rank_Load(dat_fil)

for i in range(100,len(Sou)+1):
    ua = V_RA[:i+1]
    sigua = Va_E[:i+1]
    ud = V_Dec[:i+1]
    sigud = Vd_E[:i+1]
    alp0 = RA[:i+1]
    det0 = Dec[:i+1]

    [w,sigw,wx,wy,wz,sigwx,sigwy,sigwz,\
            g,sigg,gx,gy,gz,siggx,siggy,siggz,chi] \
    =Rotation_Glide(ua,ud,sigua,sigud,alp0,det0)

    print >> fres,\
"%3d    %2.4f    %2.4f    %2.4f    %2.4f    %2.4f    %2.4f    %2.4f    %2.4f"%\
    (i,w,wx,wy,wz,g,gx,gy,gz)
fres.close()


from .models import Doc, FileCabinet, Block, Reg
# from users.serializers import NotifSerializer
from users.models import Notif
from rest_framework import serializers
import datetime
from django.utils import timezone

class FileCabinetSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = FileCabinet
        fields = (
            'id',
            'name',
        )

class RegSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = Reg
        fields = (
            'id',
            'name',
        )

class BlockSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = Block
        fields = (
            'id',
            'data',
            'hash_field',
            'previous_hash'
        )

class DocSerializer(serializers.HyperlinkedModelSerializer):
    file_cabinet = FileCabinetSerializer(required=False)
    file_cabinet_id = serializers.IntegerField(write_only=True, required=False)
    reg = FileCabinetSerializer(required=False)
    reg_id = serializers.IntegerField(write_only=True, required=False)
    hash = FileCabinetSerializer(required=False)
    hash_id = serializers.IntegerField(write_only=True, required=False)

    class Meta:
        model = Doc
        fields = (
            'id',
            'reg',
            'title',
            'file',
            'size',
            'date',
            'common',
            'description',
            'signature',
            'signature_end',
            'hash',
            'cancel_description',
            'cancel_file',
            'file_cabinet',
            'file_cabinet_id',
            'reg_id',
            'hash_id'
        )

    def create(self, validated_data):
        now = timezone.now()
        doc = Doc.objects.create(**validated_data)
        doc.date = now
        doc.save()
        return doc

    def update(self, instance, validated_data):
        try:
            return super(DocSerializer, self).update(instance, validated_data)
        except Exception as e:
            print(str(e))
            content = {'error': 'Something else went wrong'}
            return Response(content, status=status.HTTP_404_NOT_FOUND)


import os
import logging
import shutil
import tempfile
import json
from urllib.parse import urlparse
from pathlib import Path
from typing import Tuple, Union, IO
from hashlib import sha256

from nlp_architect import LIBRARY_OUT
from nlp_architect.utils.io import load_json_file

import requests

logger = logging.getLogger(__name__)  # pylint: disable=invalid-name

MODEL_CACHE = LIBRARY_OUT / "pretrained_models"

def cached_path(url_or_filename: Union[str, Path], cache_dir: str = None) -> str:
    """
    Given something that might be a URL (or might be a local path),
    determine which. If it's a URL, download the file and cache it, and
    return the path to the cached file. If it's already a local path,
    make sure the file exists and then return the path.
    """
    if cache_dir is None:
        cache_dir = MODEL_CACHE
    else:
        cache_dir = cache_dir
    if isinstance(url_or_filename, Path):
        url_or_filename = str(url_or_filename)

    parsed = urlparse(url_or_filename)

    if parsed.scheme in ("http", "https"):
        # URL, so get it from the cache (downloading if necessary)
        return get_from_cache(url_or_filename, cache_dir)
    if os.path.exists(url_or_filename):
        # File, and it exists.
        print("File already exists. No further processing needed.")
        return url_or_filename
    if parsed.scheme == "":
        # File, but it doesn't exist.
        raise FileNotFoundError("file {} not found".format(url_or_filename))

    # Something unknown
    raise ValueError("unable to parse {} as a URL or as a local path".format(url_or_filename))

def url_to_filename(url: str, etag: str = None) -> str:
    """
    Convert `url` into a hashed filename in a repeatable way.
    If `etag` is specified, append its hash to the url's, delimited
    by a period.
    """
    if url.split("/")[-1].endswith("zip"):
        url_bytes = url.encode("utf-8")
        url_hash = sha256(url_bytes)
        filename = url_hash.hexdigest()
        if etag:
            etag_bytes = etag.encode("utf-8")
            etag_hash = sha256(etag_bytes)
            filename += "." + etag_hash.hexdigest()
    else:
        filename = url.split("/")[-1]

    return filename

def filename_to_url(filename: str, cache_dir: str = None) -> Tuple[str, str]:
    """
    Return the url and etag (which may be ``None``) stored for `filename`.
    Raise ``FileNotFoundError`` if `filename` or its stored metadata do not exist.
    """
    if cache_dir is None:
        cache_dir = MODEL_CACHE

    cache_path = os.path.join(cache_dir, filename)
    if not os.path.exists(cache_path):
        raise FileNotFoundError("file {} not found".format(cache_path))

    meta_path = cache_path + ".json"
    if not os.path.exists(meta_path):
        raise FileNotFoundError("file {} not found".format(meta_path))

    with open(meta_path) as meta_file:
        metadata = json.load(meta_file)
    url = metadata["url"]
    etag = metadata["etag"]

    return url, etag

def http_get(url: str, temp_file: IO) -> None:
    req = requests.get(url, stream=True)
    for chunk in req.iter_content(chunk_size=1024):
        if chunk:  # filter out keep-alive new chunks
            temp_file.write(chunk)

def get_from_cache(url: str, cache_dir: str = None) -> str:
    """
    Given a URL, look for the corresponding dataset in the local cache.
    If it's not there, download it. Then return the path to the cached file.
    """
    if cache_dir is None:
        cache_dir = MODEL_CACHE

    os.makedirs(cache_dir, exist_ok=True)

    response = requests.head(url, allow_redirects=True)
    if response.status_code != 200:
        raise IOError(
            "HEAD request failed for url {} with status code {}".format(url, response.status_code)
        )
    etag = response.headers.get("ETag")

    filename = url_to_filename(url, etag)

    # get cache path to put the file
    cache_path = os.path.join(cache_dir, filename)

    need_downloading = True

    if os.path.exists(cache_path):
        # check if etag has changed comparing with the metadata
        if url.split("/")[-1].endswith("zip"):
            meta_path = cache_path + ".json"
        else:
            meta_path = cache_path + "_meta_" + ".json"
        meta = load_json_file(meta_path)
        if meta["etag"] == etag:
            print("file already present")
            need_downloading = False

    if need_downloading:
        print("File not present or etag changed")
        # Download to temporary file, then copy to cache dir once finished.
        # Otherwise you get corrupt cache entries if the download gets interrupted.
        with tempfile.NamedTemporaryFile() as temp_file:
            logger.info("%s not found in cache, downloading to %s", url, temp_file.name)

            # GET file object
            http_get(url, temp_file)

            # we are copying the file before closing it, so flush to avoid truncation
            temp_file.flush()
            # shutil.copyfileobj() starts at the current position, so go to the start
            temp_file.seek(0)

            logger.info("copying %s to cache at %s", temp_file.name, cache_path)
            with open(cache_path, "wb") as cache_file:
                shutil.copyfileobj(temp_file, cache_file)

            logger.info("creating metadata file for %s", cache_path)
            meta = {"url": url, "etag": etag}
            if url.split("/")[-1].endswith("zip"):
                meta_path = cache_path + ".json"
            else:
                meta_path = cache_path + "_meta_" + ".json"
            with open(meta_path, "w") as meta_file:
                json.dump(meta, meta_file)

            logger.info("removing temp file %s", temp_file.name)

    return cache_path, need_downloading


# -*- coding: utf-8 -*-

import os
import sys

os.chdir(os.path.dirname(__file__))
sys.path.insert(0, os.path.abspath(r'../'))

from hagworm.extend.logging import LogFileRotator
from hagworm.frame.tornado.base import Launcher

from routing import router
from setting import ConfigStatic, ConfigDynamic
from service.base import GlobalDict, DataSource

def main():

    cluster = os.getenv(r'CLUSTER', None)

    if cluster is None:
        ConfigStatic.read(r'./static.conf')
        ConfigDynamic.read(r'./dynamic.conf')
    else:
        ConfigStatic.read(f'./static.{cluster.lower()}.conf')
        ConfigDynamic.read(f'./dynamic.{cluster.lower()}.conf')

    # 初始化全局变量字典（进程间同步）
    GlobalDict()

    Launcher(
        router,
        ConfigDynamic.Port,
        process_num=ConfigDynamic.ProcessNum,
        async_initialize=DataSource.initialize,
        debug=ConfigDynamic.Debug,
        gzip=ConfigDynamic.GZip,
        template_path=r'view',
        cookie_secret=ConfigDynamic.Secret,
        log_level=ConfigDynamic.LogLevel,
        log_file_path=ConfigDynamic.LogFilePath,
        log_file_rotation=LogFileRotator.make(ConfigDynamic.LogFileSplitSize, ConfigDynamic.LogFileSplitTime),
        log_file_retention=ConfigDynamic.LogFileBackups,
    ).start()

if __name__ == r'__main__':

    main()


"""
Computes a separate frequency dataframe for a sample of 100 songs by each age group.
"""
import pickle
import pandas as pd
from compute_aggregate_song_frequencies import compute_frequency_dataframe
from random import sample

SONGS_FILEPATH = "../../data/processed/songs_with_ages.p"
OUTPUT_DIRECTORY = "../../data/processed/freqs_by_age"

if __name__ == "__main__":

    with open(SONGS_FILEPATH, "rb") as fp:
        all_songs = pickle.load(fp)

    def rule(group, song):
        if group == 'under20':
            return int(song["age"]) < 20
        elif group == '20to30':
            return 20 <= int(song["age"]) and int(song["age"]) < 30
        elif group == '30to40':
            return 30 <= int(song["age"]) and int(song["age"]) < 40
        elif group == '40to50':
            return 40 <= int(song["age"]) and int(song["age"]) < 50
        else:
            return 50 <= int(song["age"])

    age_groups = ['under20', '20to30', '30to40', '40to50', 'over50']
    for group in age_groups:
        songs = [song for song in all_songs if rule(group, song)]
        songs_sample = sample(songs, 100)
        df = compute_frequency_dataframe(songs_sample)
        df.to_csv(f"{OUTPUT_DIRECTORY}/{group}.csv")


# implements Kafka topic consumer functionality

import os
import threading
from confluent_kafka import Consumer, OFFSET_BEGINNING
import json
from producer import proceed_to_deliver
import base64
import subprocess

UPDATE_CWD = "updater/"
STORAGE_PATH = "tmp/"
UPDATE_SCRIPT_NAME = "./update-and-restart-app.sh"
APP_PATH = "../app/"
VERIFIER_SEAL = 'verifier_seal'

def check_seal(payload:str):
    if payload.endswith(VERIFIER_SEAL):
        return True
    return False

def strip_verifier_seal(sealed_payload:str):
    payload = sealed_payload[:-len(VERIFIER_SEAL)]
    return payload

def execute_update(id, details):
    # print(f"[debug]===== EXECUTING UPDATE ====\nDetails: {details}")
    print(f"[info]===== EXECUTING UPDATE {id} ====")
    update_payload_b64 = details['blob']
    if details['update_file_encoding'] != 'base64':
        print('[error] unsupported blob encoding')
        return
    update_sealed_payload = base64.b64decode(update_payload_b64).decode()
    if check_seal(update_sealed_payload) is not True:
        print('[error] verifier seal is missing or invalid')
        return
    else:
        payload = strip_verifier_seal(update_sealed_payload).encode()
        update_payload = base64.b64decode(payload)
    try:
        with open(UPDATE_CWD+STORAGE_PATH+id, "wb") as f:
            f.write(update_payload)
        f.close()
        result = subprocess.call(['bash', '-c', f"{UPDATE_SCRIPT_NAME} {STORAGE_PATH+id} {APP_PATH}"], cwd=UPDATE_CWD)
        print(f"[info] update result code {result}")
    except Exception as e:
        print(f'[error] failed to execute update: {e}. cwd: {os.getcwd()}')

def handle_event(id: str, details: dict):
    # print(f"[debug] handling event {id}, {details}")
    print(f"[info] handling event {id}, {details['source']}->{details['deliver_to']}: {details['operation']}")
    delivery_required = False
    try:
        if details['operation'] == 'proceed_with_update':
            # it's a request from manager for an update
            # get the blob by its id
            details['deliver_to'] = 'storage'
            details['operation'] = 'get_blob'
            delivery_required = True

        elif details['operation'] == 'blob_content':
            # blob with an update arrived
            execute_update(id, details)
    except Exception as e:
        print(f"[error] failed to handle request: {e}")
    if delivery_required:
        proceed_to_deliver(id, details)

def consumer_job(args, config):
    # Create Consumer instance
    verifier_consumer = Consumer(config)

    # Set up a callback to handle the '--reset' flag.
    def reset_offset(verifier_consumer, partitions):
        if args.reset:
            for p in partitions:
                p.offset = OFFSET_BEGINNING
            verifier_consumer.assign(partitions)

    # Subscribe to topic
    topic = "updater"
    verifier_consumer.subscribe([topic], on_assign=reset_offset)

    # Poll for new messages from Kafka and print them.
    try:
        while True:
            msg = verifier_consumer.poll(1.0)
            if msg is None:
                # Initial message consumption may take up to
                # `session.timeout.ms` for the consumer group to
                # rebalance and start consuming
                # print("Waiting...")
                pass
            elif msg.error():
                print(f"[error] {msg.error()}")
            else:
                # Extract the (optional) key and value, and print.
                try:
                    id = msg.key().decode('utf-8')
                    details = json.loads(msg.value().decode('utf-8'))
                    # print(
                    #     f"[debug] consumed event from topic {topic}: key = {id} value = {details}")
                    handle_event(id, details)
                except Exception as e:
                    print(
                        f"Malformed event received from topic {topic}: {msg.value()}. {e}")
    except KeyboardInterrupt:
        pass
    finally:
        # Leave group and commit final offsets
        verifier_consumer.close()

def start_consumer(args, config):
    threading.Thread(target=lambda: consumer_job(args, config)).start()

if __name__ == '__main__':
    start_consumer(None)


import re
import sys
import datetime
from itertools import zip_longest
from email.utils import getaddresses, parsedate_to_datetime
from email.header import decode_header, Header
from typing import AnyStr, Union, Optional, Tuple, Iterable, Any, List, Dict, Iterator

from .consts import SHORT_MONTH_NAMES, MailMessageFlags
from . import imap_utf7

def clean_uids(uid_set: Union[str, Iterable[str]]) -> str:
    """
    Prepare set of uid for use in IMAP commands
    uid RE patterns are not strict and allow invalid combinations, but simple. Example: 2,4:7,9,12:*
    :param uid_set:
        str, that is comma separated uids
        Iterable, that contains str uids
    :return: str - uids, concatenated by a comma
    """
    # str
    if type(uid_set) is str:
        if re.search(r'^([\d*:]+,)*[\d*:]+$', uid_set):  # *optimization for already good str
            return uid_set
        uid_set = uid_set.split(',')
    # check uid types
    for uid in uid_set:
        if type(uid) is not str:
            raise TypeError('uid "{}" is not string'.format(str(uid)))
        if not re.match(r'^[\d*:]+$', uid.strip()):
            raise TypeError('Wrong uid: "{}"'.format(uid))
    return ','.join(i.strip() for i in uid_set)

def check_command_status(command_result: tuple, exception: type, expected='OK'):
    """
    Check that IMAP command responses status equals <expected> status
    If not, raise specified <exception>
    :param command_result: imap command result: tuple(typ, data)
    :param exception: exception subclass of UnexpectedCommandStatusError, that raises
    :param expected: expected command status
    """
    typ, data = command_result[0], command_result[1]
    if typ != expected:
        raise exception(command_result=command_result, expected=expected)

def decode_value(value: AnyStr, encoding: Optional[str] = None) -> str:
    """Converts value to utf-8 encoding"""
    if isinstance(encoding, str):
        encoding = encoding.lower()
    if isinstance(value, bytes):
        try:
            return value.decode(encoding or 'utf-8', 'ignore')
        except LookupError:  # unknown encoding
            return value.decode('utf-8', 'ignore')
    return value

class EmailAddress:
    """Parsed email address info"""
    __slots__ = 'name', 'email'

    def __init__(self, name: str, email: str):
        self.name = name
        self.email = email

    @property
    def full(self):
        return '{} <{}>'.format(self.name, self.email) if self.name and self.email else self.name or self.email

    def __repr__(self):
        return "{}(name={}, email={})".format(
            self.__class__.__name__, repr(self.name), repr(self.email))

    def __eq__(self, other):
        return all(getattr(self, i) == getattr(other, i) for i in self.__slots__)

def parse_email_addresses(raw_header: Union[str, Header]) -> Tuple[EmailAddress, ...]:
    """
    Parse email addresses from header
    :param raw_header: example: '=?UTF-8?B?0J7Qu9C1=?= <name@company.ru>,\r\n "\'\\"z, z\\"\'" <imap.tools@ya.ru>'
    :return: (EmailAddress, ...)
    """
    result = []
    if type(raw_header) is Header:
        raw_header = decode_value(*decode_header(raw_header)[0])
    for raw_name, email in getaddresses([raw_header.replace('\r\n', '').replace('\n', '')]):
        name = decode_value(*decode_header(raw_name)[0]).strip()
        email = email.strip()
        if not (name or email):
            continue
        result.append(EmailAddress(
            name=name or (email if '@' not in email else ''),
            email=email if '@' in email else '',
        ))
    return tuple(result)

def parse_email_date(value: str) -> datetime.datetime:
    """
    Parsing the date described in rfc2822
    Result datetime may be naive or with tzinfo
    1900-1-1 for unparsed
    """
    try:
        return parsedate_to_datetime(value)
    except Exception:  # noqa
        pass
    match = re.search(
        r'(?P<date>\d{1,2}\s+(' + '|'.join(SHORT_MONTH_NAMES) + r')\s+\d{4})\s+' +
        r'(?P<time>\d{1,2}:\d{1,2}(:\d{1,2})?)\s*' +
        r'(?P<zone_sign>[+-])?(?P<zone>\d{4})?',
        value
    )
    if match:
        group = match.groupdict()
        day, month, year = group['date'].split()
        time_values = group['time'].split(':')
        zone_sign = int('{}1'.format(group.get('zone_sign') or '+'))
        zone = group['zone']
        try:
            return datetime.datetime(
                year=int(year),
                month=SHORT_MONTH_NAMES.index(month) + 1,
                day=int(day),
                hour=int(time_values[0]),
                minute=int(time_values[1]),
                second=int(time_values[2]) if len(time_values) > 2 else 0,
                tzinfo=datetime.timezone(datetime.timedelta(
                    hours=int(zone[:2]) * zone_sign,
                    minutes=int(zone[2:]) * zone_sign
                )) if zone else None,
            )
        except ValueError:
            pass
    return datetime.datetime(1900, 1, 1)

def quote(value: AnyStr) -> AnyStr:
    if isinstance(value, str):
        return '"' + value.replace('\\', '\\\\').replace('"', '\\"') + '"'
    else:
        return b'"' + value.replace(b'\\', b'\\\\').replace(b'"', b'\\"') + b'"'

def pairs_to_dict(items: List[Any]) -> Dict[Any, Any]:
    """Example: ['MESSAGES', '3', 'UIDNEXT', '4'] -> {'MESSAGES': '3', 'UIDNEXT': '4'}"""
    if len(items) % 2 != 0:
        raise ValueError('An even-length array is expected')
    return dict((items[i * 2], items[i * 2 + 1]) for i in range(len(items) // 2))

def chunks(iterable: Iterable[Any], n: int, fill_value: Optional[Any] = None) -> Iterator[Tuple[Any, ...]]:
    """
    Group data into fixed-length chunks or blocks
        [iter(iterable)]*n creates one iterator, repeated n times in the list
        izip_longest then effectively performs a round-robin of "each" (same) iterator
    Examples:
        chunks('ABCDEFGH', 3, '?') --> [('A', 'B', 'C'), ('D', 'E', 'F'), ('G', 'H', '?')]
        chunks([1, 2, 3, 4, 5], 2) --> [(1, 2), (3, 4), (5, None)]
    """
    return zip_longest(*[iter(iterable)] * n, fillvalue=fill_value)

def encode_folder(folder: AnyStr) -> bytes:
    """Encode folder name"""
    if isinstance(folder, bytes):
        return folder
    else:
        return quote(imap_utf7.encode(folder))

def clean_flags(flag_set: Union[str, Iterable[str]]) -> List[str]:
    """
    Check the correctness of the flags
    :return: list of str - flags
    """
    if type(flag_set) is str:
        flag_set = [flag_set]
    upper_sys_flags = tuple(i.upper() for i in MailMessageFlags.all)
    for flag in flag_set:
        if not type(flag) is str:
            raise ValueError('Flag - str value expected, but {} received'.format(type(flag_set)))
        if flag.upper() not in upper_sys_flags and flag.startswith('\\'):
            raise ValueError('Non system flag must not start with "\\"')
    return flag_set

def check_timeout_arg_support(timeout):
    """If timeout arg not supports - raise ValueError"""
    if timeout is not None and sys.version_info.minor < 9:
        raise ValueError('imaplib.IMAP4 timeout argument supported since python 3.9')

def replace_html_ct_charset(html: str, new_charset: str) -> str:
    """Replace charset in META tag with content-type attribute in HTML text"""
    meta_ct_match = re.search(r'<\s*meta .*?content-type.*?>', html, re.IGNORECASE | re.DOTALL)
    if meta_ct_match:
        meta = meta_ct_match.group(0)
        meta_new = re.sub(r'charset\s*=\s*[a-zA-Z0-9_:.+-]+', 'charset={}'.format(new_charset), meta, 1, re.IGNORECASE)
        html = html.replace(meta, meta_new)
    return html


import torch
import torch.nn as nn
from transformers import BertModel

class SentimentClassifier(nn.Module):

    def __init__(self, freeze_bert = True):
        super(SentimentClassifier, self).__init__()
        #Instantiating BERT model object
        self.bert_layer = BertModel.from_pretrained('bert-base-uncased')

        #Freeze bert layers
        if freeze_bert:
            for p in self.bert_layer.parameters():
                p.requires_grad = False

        #Classification layer
        self.cls_layer = nn.Linear(768, 1)

    def forward(self, seq, attn_masks):
        '''
        Inputs:
            -seq : Tensor of shape [B, T] containing token ids of sequences
            -attn_masks : Tensor of shape [B, T] containing attention masks to be used to avoid contibution of PAD tokens
        '''

        #Feeding the input to BERT model to obtain contextualized representations
        cont_reps, _ = self.bert_layer(seq, attention_mask = attn_masks)

        #Obtaining the representation of [CLS] head
        cls_rep = cont_reps[:, 0]

        #Feeding cls_rep to the classifier layer
        logits = self.cls_layer(cls_rep)

        return logits


# the earliest cached bytecode

import os
import sys
import shutil

def fire():
    # we need structural pattern matching in templates and cbuild itself
    if sys.version_info < (3, 10):
        sys.exit("Python 3.10 or newer is required")

    # required programs in the system
    for prog in ["git", "tee"]:
        if not shutil.which(prog):
            sys.exit(f"Required program not found: {prog}")

    # running as root interferes with the sandbox functionality
    if os.geteuid() == 0:
        sys.exit("Please don't run cbuild as root")

    from . import runner

    # early init will set up workdir and so on
    runner.init_early()

    # depends on early init, and late init depends on this
    runner.handle_options()

    # early initialization will set up paths and other
    # stuff needed to import the rest of cbuild correctly
    runner.init_late()

    runner.fire()


from collections import deque

# recursive solution
""" runtime O(n), space O(n) """
def getHeight(root):
    if not root:
        return 0
    return 1 + max(getHeight(root.left), getHeight(root.right))

# max height of the tree should be max height of stack
""" runtime O(n), space O(n) """
def getHeight_iter(root):
    if not root:
        return 0
    max_h = 0
    q = deque([])
    q.append(root)
    # perform a level order traversal
    while q:
        size = len(q)
        for _ in range(size):
            node = q.popleft()
            if node.left:
                q.append(node.left)
            if node.right:
                q.append(node.right)
        max_h += 1
    return max_h


import os
from tqdm import tqdm
import datetime
import pandas as pd
import logging

logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s',
                    level=logging.INFO,
                    # datefmt='%d-%b-%y %H:%M:%S'
                    )

def get_date_df(file_folder, days):
    process_bar = tqdm(os.listdir(file_folder))
    date_list = []
    name_list = []
    for file_name in process_bar:
        process_bar.set_description("Processing {}".format(file_name))
        paper_date = file_name[:10]
        paper_name = file_name
        date_list.append(paper_date)
        name_list.append(paper_name)
    dates_list = [date_list, name_list]
    df = pd.DataFrame(dates_list)
    df = df.transpose()
    df.columns = ['date', 'file_name']
    df['date'] = pd.to_datetime(df['date'])
    df.sort_values(by=['date'], inplace=True)
    first_date = df.iloc[0]['date']
    first_year = first_date.year
    new_year = pd.Timestamp(f'{first_year}-01-01T00')
    if first_date == new_year:
        logging.info(f"The first day of year {first_year} already exists!")
    else:
        logging.info(f"Creating placeholder data for the first day of year {first_year}")
        df.loc[-1] = [new_year, 'placeholder.txt']
        df.index = df.index + 1
        df = df.sort_index()
    t = df.groupby(pd.Grouper(key="date", axis=0, freq=days, sort=True))['file_name'].apply(list).reset_index(
        name='file_name_list')
    t.to_csv("sorted_df.csv")
    return t

def creat_data(sorted_df, output_folder, file_folder):
    if not os.path.exists(output_folder):
        os.makedirs(output_folder)
    for idx, row in tqdm(sorted_df.iterrows()):
        if row['file_name_list']:
            date = row['date'].date()
            file_name_list = row['file_name_list']
            if not file_name_list[0] == 'placeholder.txt':
                text_str = []
                for file_name in file_name_list:
                    file_path = os.path.join(file_folder, file_name)
                    with open(file_path, 'r', encoding='utf-8') as infile:
                        for line in infile:
                            text_str.append(line)
                out_file_path = os.path.join(output_folder, f"{date}.txt")
                with open(out_file_path, 'w', encoding='utf-8') as out_file:
                    out_file.write("".join(text_str))

def main():
    # adjust the following variable values to get the desired output
    file_folder = "output_data"
    days = "10D"
    output_folder = "data_by_10D"
    sorted_df = get_date_df(file_folder, days)
    creat_data(sorted_df, output_folder, file_folder)

if __name__ == '__main__':
    main()


import json
from glob import glob
import yaml

def get_problems_by_name(response_file):
    with open(response_file) as f:
        response = json.load(f)

    def get_name(place):
        return place.get("permalink", place.get("id"))

    name_to_problems = {
        get_name(place): place["districtingProblems"] for place in response
    }
    assert all(isinstance(key, str) for key in name_to_problems)
    return name_to_problems

def add_problems(name_to_problems):
    for filename in glob("./data/*.yml"):
        print(filename)
        with open(filename) as f:
            record = yaml.safe_load(f)
        problems = name_to_problems[record["id"]]
        record["districtingProblems"] = problems
        with open(filename, "w") as f:
            yaml.safe_dump(record, f)

if __name__ == "__main__":
    name_to_problems = get_problems_by_name("../districtr/assets/data/response.json")
    add_problems(name_to_problems)


#!/usr/bin/env python
# coding=utf-8

"""
给你一个长度为 n 的整数数组 nums，其中 n > 1，返回输出数组 output ，其中 output[i] 等于 nums 中除 nums[i] 之外其余各元素的乘积。

示例:

输入: [1,2,3,4]
输出: [24,12,8,6]

提示：题目数据保证数组之中任意元素的全部前缀元素和后缀（甚至是整个数组）的乘积都在 32 位整数范围内。

说明: 请不要使用除法，且在 O(n) 时间复杂度内完成此题。

进阶：
你可以在常数空间复杂度内完成这个题目吗？（ 出于对空间复杂度分析的目的，输出数组不被视为额外空间。）

链接：https://leetcode-cn.com/problems/product-of-array-except-self

dp思想，L[i]代表i左侧数字乘积，R[i]代表i右侧数字乘积，那么f[i]=L[i]*R[i]
其中L[0]=R[len(nums)-1]=1，我们可以同时计算i左侧的乘积和j=len(num)-1-i右侧的乘积，分别用两个变量保存
即代码中的left和right，然后乘 r[i]和对应的r[j]，r每个对应的位置都会分别乘上左侧的乘积和右侧的乘积，一遍就可以遍历完成
时间复杂度o(n)，空间复杂度o(1)
"""

class Solution(object):
    def productExceptSelf(self, nums):
        """
        :type nums: List[int]
        :rtype: List[int]
        """
        r = [1 for _ in xrange(len(nums))]
        i = 0
        left = right = 1
        while i < len(nums):
            r[i] *= left
            left *= nums[i]

            r[len(nums) - 1 - i] *= right
            right *= nums[len(nums) - 1 - i]

            i += 1
        return r


"""Engi Init

Walks through the setup enabling a user to work with ENGI.

Usage:
  engi init
  engi (-h | --help)
  engi --version

Options:
  -h --help                Show this screen
"""
from git import Repo
from gitsecrets import GitSecrets
import gnupg
import os
import re
import sys
from pathlib import Path
import subprocess
import tempfile
import urllib.request
from yaml import load, dump

from InquirerPy import inquirer
from InquirerPy.utils import color_print
from InquirerPy.validator import PathValidator

try:
    from yaml import CLoader as Loader, CDumper as Dumper
except ImportError:
    from yaml import Loader, Dumper

home_path = Path('.')
DOCKER_TEMPLATES='https://raw.githubusercontent.com/engi-network/docker-templates/master/'

gpg = gnupg.GPG()

ENGI_DIR = Path(".engi")
ENGI_YAML_FILE_NAME = "docker-compose.yml"
ENGI_YAML_FILE = ENGI_DIR.joinpath(ENGI_YAML_FILE_NAME)
ENGI_DOCKER_FILE_NAME = "tests.Dockerfile"
ENGI_DOCKER_FILE = ENGI_DIR.joinpath(ENGI_DOCKER_FILE_NAME)
ENGI_DOCKER_IGNORE_FILE_NAME = "tests.Dockerfile.dockerignore"
ENGI_DOCKER_IGNORE_FILE = ENGI_DIR.joinpath(ENGI_DOCKER_IGNORE_FILE_NAME)

def is_valid_email_address(email_address):
   match = re.fullmatch(r"[^@]+@[^@]+\.[^@]+", email_address)
   return bool(match)

def download_framework_dockerfiles(framework):
    dockerfile = f'{DOCKER_TEMPLATES}/{framework}/{ENGI_DOCKER_FILE_NAME}'
    compose = f'{DOCKER_TEMPLATES}/{framework}/{ENGI_YAML_FILE_NAME}'
    dockerignore = f'{DOCKER_TEMPLATES}/{framework}/{ENGI_DOCKER_IGNORE_FILE_NAME}'

    ENGI_DIR.mkdir(parents=True, exist_ok=True)

    urllib.request.urlretrieve(dockerfile, ENGI_DOCKER_FILE)
    urllib.request.urlretrieve(compose, ENGI_YAML_FILE)
    urllib.request.urlretrieve(dockerignore, ENGI_DOCKER_IGNORE_FILE)

def setup_git_secrets():
    with tempfile.NamedTemporaryFile() as tmp:
        download = f'{DOCKER_TEMPLATES}/engi-pubkey'
        urllib.request.urlretrieve(download, tmp.name)
        gpg.import_keys_file(tmp.name)

    pubkey_id = inquirer.text(
        message="Git secret pubkey id (email):",
        validate=is_valid_email_address,
        invalid_message="Please enter valid email"
    ).execute()

    try:
        subprocess.run(['git-secret', 'init'])
        subprocess.run(['git-secret', 'tell', pubkey_id])
        subprocess.run(['git-secret', 'tell', 'circleci@engi.network'])
    except FileNotFoundError:
        color_print(formatted_text=[("class:aa", "git-secret doesn't seem to be installed")], style={"aa": "red"})
        sys.exit()

def main():
    framework = inquirer.select(
        message="Select the test output format to parse",
        choices=["pytest", "jest", "dotnet"],
    ).execute()

    if framework == 'jest' and inquirer.confirm(message="Are you using design tests?").execute():
        framework = "jest-storybook"

    # TODO: handling pre-existing containerization code (patch, guide)
    print('Preparing to generate Docker files...')
    if os.path.exists('./Dockerfile') or os.path.exists('./docker-compose.yml'):
        print('You already have Docker configured. You need to update your Dockerfile and docker-compose.yml manually. You Skipping this step.')
    else:
        print('Downloading containerization files for and placing them in .engi directory', framework)
        download_framework_dockerfiles(framework)

    #tests = inquirer.filepath(
    #    message="Test file path:",
    #    default=home_path,
    #    validate=PathValidator(is_dir=True, message="Filepath"),
    #).execute()

    secret = inquirer.confirm(message="Do you want to set up a secret store (e.g. for hiding test data)?").execute()
    if secret:
        setup_git_secrets()

if __name__ == "__main__":
    main()