import os, sys, time, math, random

import numpy as np

import numpy.ma as ma

import geopandas as gpd
import rasterio as rio

import rasterio.features as rfeat
import rasterio.windows as rwin

import shapely.geometry as shp

from lib.annotators.Annotator import Annotator, anno_kwargs_key

"""
New candidate selection system to work with single datasets at a time.

Annotating in complete lines instead of greedy coordinate point selection.

"""

cma_kwargs_key = {
    'annotator': ("(Annotator [None])", "Annotator to configure parent parameters by. If no annotator is passed, will use annotator default."),
    'pairs': ("(int [10])", "Number of coordinate pairs to be generated for each candidate grouping."),
    'off_dist': ("(int, meters [1])", "Base distance  to offset candidate coordinate pairs.")
}

cma_kwargs_key.update(("Annotator - {}".format(key), item) for key, item in anno_kwargs_key.items())

class Dynamic_Preloading_Annotator(Annotator):
    """
    Uses a predicted class map represented by a rasterio.DatasetReader raster
    to improve shapely.geometry.Linestring annotations.

    Each set of candidate coordinates held in a shapely.geometry.MultiPoint,
    each Linestring's total candidates represented by a gpd.GeoDataFrame.

    Candidate Generation and weighting configuration stored in class attributes.

    Attributes:
        pairs (int):        Number of coordinate pairs to be generated
                                on either side of source annotation.
        off_dist (int):     Offset distance for each candidate point.
        outer_pairs (int):  Number of coordinate pairs to be generated
                                with multiplicative increment.
        outer_multiplier (float):   Multiplier used to increment outer pairs.
        minimum_weight (float):     Minimum weight for candidate update
        coordinate_precision (float):  Decimal coordinate_precision used to compare coordinates.
        verbosity (0,1):    Determines verbosity for printing indexes.
        use_origin_default (bool [True]):   If True, always use origin as default candidate.
                                            If False, use previous index as default candidate

        random_select_chance (float [0.02]):    Chance to use random candidate selection for each candidate group.
        random_select_range (int [4]):          Distance from top coordinate point that a random candidate may be selected.

    """
    def __init__(self, **kwargs):
        """
        Initialize class map annotator.

        Notes:
            If annotator is passed in keys, update parameters from that annotator's configuration. Otherwise expects the passed parameters.

        """

        # Initialize annotator
        if 'annotator' in kwargs.keys():
            super().__init__(**kwargs['annotator'].__dict__)
        else:
            super().__init__(**kwargs)

        # Default Attributes
        default_attr = dict(
            pairs = 10,                 # pairs to be generated on either side linearly.
            off_dist = 1,               # offset dist incrementable (meters)
            coordinate_precision = 1e-4,# Float precision for coordinate comparison
            weight_buffer = 3,          # Buffer to apply to candidate linestrings when weighting
            L = 0.001,                  # weight for line length consideration when weighting canididates
            min_p = 1e-06,              # minimum valid probability
            normalize_full = False,     # Optionally normalize length over K^2 candidates
            kwargs_key = cma_kwargs_key
        )

        # Allowed Attributes
        allowed_attr = list(default_attr.keys())

        # Update with passed arguments
        default_attr.update(kwargs)

        # Update kwargs
        self.__dict__.update((k,v) for k,v in default_attr.items() if k in allowed_attr)

    ### Visualization and Information

    def get_candidate_range(self):
        """ Returns the maximum range that a candidate may be generated. Ceiling function applied to final value. (Accurate) """

        dist = 0

        for i in range(self.pairs + self.outer_pairs):

            # increment
            dist += self.off_dist

            # if outer_pairs are passed, exponentially apply increase
            if i > self.pairs:
                dist += (self.outer_multiplier*(i-self.pairs))
                dist = np.round(dist, 4)

        return int(np.ceil(dist))

    ### CANDIDATE GENERATION

    def _point_set(self, point, prev_p, next_p):
        """
        Generates a set of candidate points for a single coordinate point.

        Saves offset from source annotation, returning points and respective offset distance as dict.

        Args:
            point (shp.Point):  Target point Designed to generate candidate set.
            prev_p (shp.Point): Previous point in annotation line sequence. Used to determine slope.
            next_p (shp.Point): Next point in annotation line sequence. Used to determine slope.

        Returns:

            shp.MultiPoint: holds all candidate points for the source point.
                            The first point in the multipoint is always the source,
                            points increase in distance from the source after that.
        """

        # Get offset weights
        off_x, off_y = self.get_offsets(prev_p, next_p)

        # Initialize set with source point
        candidates = [point]

        # Initialize offset distance
        dist = 0

        for i in range(self.pairs):

            # increment
            dist += self.off_dist

            # Create coordinate offsets for each axis
            x_dist = off_x * dist
            y_dist = off_y * dist

            # Append points and their respective offset tuples
            candidates.append(shp.Point(((point.x + x_dist), (point.y - y_dist))))
            candidates.append(shp.Point(((point.x - x_dist), (point.y + y_dist))))

        # Filter out geometries that do not lie on raster
        for idx, point in enumerate(candidates):
            if not self.crop_window.contains(point):
                del candidates[idx]

        # VERBOSITY
        if self.verbosity == 1:
            self.coordinate_index += 1
            print("created new coordinate group, index:", self.coordinate_index)

        # May remove, empty sets should never appear if other systems work properly
        if len(candidates) == 0:
            print("WARNING: Empty set of candidate coordinates created.")
            return None

        # Convert list of points to shp.MultiPoint and return
        return shp.MultiPoint(candidates)

    def _generate_points(self, line):
        """ Generates candidate groups over a line from internal configuration.

        Args:
            line (shp.Linestring): Linestring to generate candidate points over.

        Returns:
            list(shp.MultiPoint): list of candidate point groups in set,
                                  including origin and terminating groups.
        """

        if not isinstance(line, shp.LineString):
            print("Annotator Error (generate_points): Invalid source line type, expected: {}       Passed: {}\n\n".format(type(shp.LineString), type(line)))

        origin = shp.Point(line.coords[0])
        target_point = line.interpolate(self.interval)

        # Initialize list of candidate groups with origin candidates.
        candidate_groups = [self._point_set(origin, origin, target_point)]

        i = 0
        while True:
            # prev_p starts as origin, target point starts as index 1
            prev_p, this_p, next_p = line.interpolate(self.interval*i), line.interpolate(self.interval*(i+1)), line.interpolate(self.interval*(i+2))
            i += 1

            candidate_groups.append(self._point_set(this_p, prev_p, next_p))

            # if the current target point has the same coordinates as the endpoint of the line, break
            if self.same_coords(this_p.coords[0], line.coords[-1]):
                break

        # VERBOSITY
        if self.verbosity == 1:
            self.line_index += 1
            print("Coordinate Set generation complete, index:", self.line_index)
            print("Total coordinate sets created:", self.coordinate_index)
            self.coordinate_index = 0

        # filter empty groups
        candidate_groups = [x for x in candidate_groups if x != None]
        return candidate_groups

    ### CANDIDATE SELECTION

    def get_length_range(self, group_a, group_b):
        """ Get the min and max candidate line lengths slowly accross k^2 candidates.

        Args:
            group_a: shp.MultiPoint of target
            group_a: shp.MultiPoint of previous
        Returns:
            min_length (float), max_length (float)
        """

        l_max, l_min = 0,0
        for p_idx, pt in enumerate(group_a):
            lengths = [pt.distance(other_pt) for other_pt in group_b]
            if p_idx > 0:
                lengths += [l_max, l_min]
            l_max = np.amax(lengths)
            l_min = np.amin(lengths)

        return l_min, l_max

    def origin_state(self, origin_group, class_map):
        """ Assigns weights to origin group from internal predicted class map

        Args:
            origin_group (shp.MultiPoint): Group of points to select origin point from.
            class_map (rio.DatasetReader): Predicted class map as raster to read weights from.

        Returns:
            int: optimal starting point's index based on predicted class map weights.
        """

        # Initialize weight list
        weights = []

        origin_points = list(origin_group)

        # Buffer points into polygons for evaluation
        buffed_points = [point.buffer(self.weight_buffer) for point in origin_group]

        for idx, point in enumerate(buffed_points):
            if not self.crop_window.contains(point):
                del origin_points[idx]
                del buffed_points[idx]

        # Window to read candidate group from raster
        group_window = rfeat.geometry_window(class_map, buffed_points)

        # read pixel data from class map using geometry window
        group_data = self.class_map.read(1, window=group_window)

        for idx, candidate_origin in enumerate(buffed_points):

            # Create a boolean mask representing the linestring from the windowed data for this candidate group.
            mask = rfeat.geometry_mask([candidate_origin], out_shape=group_data.shape, transform=rwin.transform(group_window, class_map.transform), all_touched=True)

            # Apply boolean mask to raster's data
            masked_data = ma.array(group_data, mask=mask)

            # Calculate mean weight for segment
            weights.append(masked_data.sum() / ma.compressed(masked_data).shape[0])

        origin_state = {
            'xy': [pt.coords[0] for pt in origin_points],
            'parent_idx': None,
            'weight': weights
        }
        return origin_state

    def get_state_row(self, class_map, target_group, prev_group, weights: list):
        """ Gets a row of state with this set's parent nodes and  their values. """

        candidate_line_sets = []

        # Stores state for candidate group
        state = {
            'xy': [],
            'parent_idx': [],
            'weight': [],
            # NOTE: Future terms.
            # - Shape of line
            # - Direction of shift
            # - Neighboring line segment shapes
            # - - **Similarity in shape between neighboring lines should be highly weighted.
        }

        # Concatenate and buffer all points to create a window containing all needed class map data
        all_points = list(target_group) + list(prev_group)
        group_window = rfeat.geometry_window(class_map, [pt.buffer(self.weight_buffer) for pt in all_points])

        # read pixel data from class map using geometry window
        group_data = class_map.read(1, window=group_window)

        # Get terms for normalization
        if self.normalize_full:
            l_min, l_max = self.get_length_range(target_group, prev_group)

        # Iterate over target group's points
        for anchor_idx, anchor in enumerate(target_group):

            # Reset top index and weight
            top_weight, top_idx = 0,0

            # Create candidate lines connecting this anchor point to every previous point
            candidate_lines = [shp.LineString([anchor, point]).buffer(self.weight_buffer) for point in prev_group.geoms]

            # Get local normalization term
            if not self.normalize_full:
                l_min, l_max = self.get_length_range([anchor], prev_group)

            for idx, (candidate_line, weight_value) in enumerate(zip(candidate_lines, weights)):
                # Create a boolean mask representing the linestring from the windowed data for this candidate group.
                mask = rfeat.geometry_mask([candidate_line], out_shape=group_data.shape, transform=rwin.transform(group_window, class_map.transform), all_touched=True)

                # Apply boolean mask to raster's data
                masked_data = ma.array(group_data, mask=mask)

                # Calculate mean weight for segment
                weight = masked_data.sum() / ma.compressed(masked_data).shape[0]

                # Check weight by threshold
                if weight <= self.min_p:
                    weight = 0.0

                # Calculate line length, normalizing by previously evaluated terms.
                line_length = ((candidate_line.length - l_min) / (l_max - l_min))
                length_value = (self.L * (1 - line_length))
                weight += length_value

                if (idx == 0) or ((weight_value + weight) > top_weight):
                    top_weight = (weight_value + weight)
                    top_idx = idx

            # Update state matrix
            state['xy'].append(anchor.coords[0])
            state['parent_idx'].append(top_idx)
            state['weight'].append(top_weight)

        return state

    def _new_annotation(self, line, class_map=None):
        """
        Create a new annotation from a Linestring and predicted class map
        using internal parameters.

        Args:
            line (shp.LineString): Source annotation to be improved in same CRS map raster.
            class_map (rio.DatasetReader): Predicted class map used to evaluate candidates.
            use_default (bool): Optionally continue annotations with zero-weight candidates by taking the same index point as the previous
        Returns:
            list( (float, float) ): (x, y) coordinate offset tuples from the source annotation
            shp.Linestring: Improved annotation
        """

        if not isinstance(line, shp.LineString):
            raise TypeError("Class_Map_Annotator.new_annotation(): Expected type '{}' for line, recieved: '{}'".format(shp.LineString.__name__, type(line).__name__))

        if (class_map == None) and (self.class_map == None):
            raise RuntimeError("Class_Map_Annotator.new_annotation(): No class map to weight candidates.")

        self.class_map = class_map

        # Generate candidate point groups
        candidate_groups = self._generate_points(line)

        state_matrix = [self.origin_state(candidate_groups[0], class_map=class_map)]

        for i in range(1, len(candidate_groups)):
            state_matrix.append(self.get_state_row(class_map=class_map, target_group=candidate_groups[i], prev_group=candidate_groups[i-1], weights=state_matrix[i-1]['weight']))

        return state_matrix

    def preload_candidates(self, gdf, class_map):
        """
        Generate only the candidate coordinate set for visualization,
        optionally clip to raster.

        Args:
            gdf (gpd.GeoDataFrame): Source annotation to generate candidates from.
            class_map (rio.DatasetReader): Optional raster to clip candidate geometries.
            flatten (bool, default=True): Option to flatten output geometries
                True: Return candidates as a single GeoDataFrame of all candidate points. (Default)
                or
                False: Return candidates as a list of GeoDataFrames (one per geometry in input).
            out_path (filepath): Optional filepath to save candidates. If flatten==False, out_path is ignored.

        Returns:
            gpd.GeoDataFrame: Holds candidate groups as MultiPoints in source CRS. (flatten=True)
            or
            list(gpd.GeoDataFrame): One GeoDataFrame of candidate points for each Linestring in source. (flatten=False)
        """

        self.class_map = class_map

        # set window for cropping geoms and clip geometries
        self.set_crop_window(class_map)
        gdf = gdf.to_crs(class_map.crs)
        gdf = gpd.clip(gdf, mask=self.crop_window, keep_geom_type=True)

        all_line_data = []
        for idx, line in enumerate(gdf.geometry):
            line_data = {
                'windows': [],
                'candidates': [],
            }

            # Get All candidate groups across the line
            candidate_groups = self._generate_points(line)

            # Focus on origin
            target_group = list(candidate_groups[0])
            last_buffed_points = [point.buffer(self.weight_buffer) for point in target_group]

            for d_idx, point in enumerate(last_buffed_points):
                if not self.crop_window.contains(point):
                    del target_group[d_idx]
                    del last_buffed_points[d_idx]

            # Add origin window to this line
            origin_window =rfeat.geometry_window(class_map, last_buffed_points)

            # Get origin masks with simple weight buffer
            origin_masks = [rfeat.geometry_mask([point], out_shape=(origin_window.height, origin_window.width), transform=rwin.transform(origin_window, class_map.transform), all_touched=True) for point in last_buffed_points]

            # update list of candidate dicts with origin data
            line_data['candidates'].append({'geoms': target_group, 'lengths': [], 'masks': origin_masks})
            line_data['windows'].append(origin_window)

            # Iterate over other candidate groups for this line
            for i in range(1, len(candidate_groups)):

                # Clarify group indexes
                prev_group = list(candidate_groups[i-1])
                target_group = list(candidate_groups[i])

                # Initialize lists
                candidate_lengths, candidate_masks = [], []

                # Buffer these points to get a reference candidate window
                buffed_points = [point.buffer(self.weight_buffer) for point in target_group]
                for d_idx, point in enumerate(buffed_points):
                    if not self.crop_window.contains(point):
                        del target_group[d_idx]
                        del buffed_points[d_idx]

                group_window = rfeat.geometry_window(class_map, (buffed_points+last_buffed_points))

                # Iterate over candidate points in each group
                for anchor in target_group:
                    candidate_lines = [shp.LineString([anchor, point]) for point in prev_group]
                    candidate_lengths.append([line.length for line in candidate_lines])

                    candidate_lines = [line.buffer(self.weight_buffer) for line in candidate_lines]
                    candidate_masks.append([rfeat.geometry_mask([line], out_shape=(group_window.height, group_window.width), transform=rwin.transform(group_window, class_map.transform), all_touched=True) for line in candidate_lines])

                line_data['candidates'].append({'geoms': target_group, 'lengths': candidate_lengths, 'masks': candidate_masks})
                line_data['windows'].append(group_window)

                # Reset buffed points
                last_buffed_points = buffed_points

                #print("- Finished Group {} of {} (Line {}).".format(i, len(candidate_groups)-1, idx))

            print("Completed Line {} of {}.\n".format(idx, len(gdf.geometry)-1))
            all_line_data.append(line_data)

        return all_line_data

    def back_prop(self, state_matrix) -> list:
        """ Converts state matrix to updated linestring. """

        # Holds updated shapely points
        new_points = []

        top_weight = 0
        top_idx = 0
        for idx, weight in enumerate(state_matrix[-1]['weight']):
            if weight > top_weight:
                top_weight = weight
                top_idx = idx

        i = 1
        while i <= len(state_matrix):
            new_points.insert(0, shp.Point(state_matrix[-i]['xy'][top_idx]))

            # Final points
            if state_matrix[-i]['parent_idx'] == None: break

            top_idx = state_matrix[-i]['parent_idx'][top_idx]
            i += 1
        return shp.LineString(new_points)

    def origin_state_preloaded(self, origin_data, window, class_map):
        """ Assigns weights to origin group from internal predicted class map

        Args:
            origin_group (shp.MultiPoint): Group of points to select origin point from.
            class_map (rio.DatasetReader): Predicted class map as raster to read weights from.

        Returns:
            int: optimal starting point's index based on predicted class map weights.
        """

        # Initialize weight list
        weights = []

        # read pixel data from class map using geometry window
        group_data = class_map.read(1, window=window)

        for idx, mask in enumerate(origin_data['masks']):

            # Apply boolean mask to raster's data
            masked_data = ma.array(group_data, mask=mask)

            # Calculate mean weight for segment
            weights.append(masked_data.sum() / ma.compressed(masked_data).shape[0])

        origin_state = {
            'xy': [pt.coords[0] for pt in origin_data['geoms']],
            'parent_idx': None,
            'weight': weights
        }
        return origin_state

    def get_state_row_preloaded(self, class_map, group_data, window, weights):
        """ Gets a row of state with this set's parent nodes and  their values. """

        candidate_line_sets = []

        # Stores state for candidate group
        state = {
            'xy': [],
            'parent_idx': [],
            'weight': [],
        }

        group_window = window

        # read pixel data from class map using geometry window
        group_arr = class_map.read(1, window=group_window)

        # Get terms for normalization
        if self.normalize_full:
            all_lens = [l for lengths in group_data['lengths'] for l in lengths]
            l_min, l_max = np.amin(all_lens), np.amax(all_lens)

        # Iterate over target group's points
        for anchor_idx, (anchor, anchor_lengths, anchor_mask_list) in enumerate(zip(group_data['geoms'], group_data['lengths'], group_data['masks'])):

            # Reset top index and weight
            top_weight, top_idx = 0,0

            if not self.normalize_full:
                l_min, l_max = np.amin(anchor_lengths), np.amax(anchor_lengths)

            for idx, (p_length, mask, weight_value) in enumerate(zip(anchor_lengths, anchor_mask_list, weights)):

                # Apply boolean mask to raster's data
                masked_data = ma.array(group_arr, mask=mask)

                # Calculate mean weight for segment
                weight = masked_data.sum() / ma.compressed(masked_data).shape[0]

                # Check weight by threshold
                if weight <= self.min_p:
                    weight = 0.0

                # Calculate line length, normalizing by previously evaluated terms.
                line_length = ((p_length - l_min) / (l_max - l_min))
                length_value = (self.L * (1 - line_length))
                weight += length_value

                if (idx == 0) or ((weight_value + weight) > top_weight):
                    top_weight = (weight_value + weight)
                    top_idx = idx

            # Update state matrix
            state['xy'].append(anchor.coords[0])
            state['parent_idx'].append(top_idx)
            state['weight'].append(top_weight)

        return state

    def new_anno_preloaded(self, line_data: dict, class_map=None):
        """
        Create a new annotation from a Linestring and predicted class map
        using internal parameters.

        Args:
            line (shp.LineString): Source annotation to be improved in same CRS map raster.
            class_map (rio.DatasetReader): Predicted class map used to evaluate candidates.
            use_default (bool): Optionally continue annotations with zero-weight candidates by taking the same index point as the previous
        Returns:
            list( (float, float) ): (x, y) coordinate offset tuples from the source annotation
            shp.Linestring: Improved annotation
        """

        if (class_map == None) and (self.class_map == None):
            raise RuntimeError("Class_Map_Annotator.new_annotation(): No class map to weight candidates.")

        self.class_map = class_map

        state_matrix = []

        for idx, (candidate_data, window) in enumerate(zip(line_data['candidates'], line_data['windows'])):
            if idx == 0:
                # Origin
                state_matrix.append(self.origin_state_preloaded(class_map=class_map, origin_data=candidate_data, window=window))
            else:
                state_matrix.append(self.get_state_row_preloaded(class_map=class_map, group_data=candidate_data, window=window, weights=state_matrix[idx-1]['weight']))

        return self.back_prop(state_matrix)

    def update_gdf_from_preload(self, all_data, class_map, out_path = None):

        lines = []

        for line_data in all_data:
            lines.append(self.new_anno_preloaded(line_data, class_map))

        annotation = gpd.GeoDataFrame(geometry=lines, crs=class_map.crs)

        if out_path:
            annotation.to_file(out_path)

        return annotation

    def get_candidates(self, gdf, class_map=None, flatten=True, out_path=None):
        """
        Generate only the candidate coordinate set for visualization,
        optionally clip to raster.

        Args:
            gdf (gpd.GeoDataFrame): Source annotation to generate candidates from.
            class_map (rio.DatasetReader): Optional raster to clip candidate geometries.
            flatten (bool, default=True): Option to flatten output geometries
                True: Return candidates as a single GeoDataFrame of all candidate points. (Default)
                or
                False: Return candidates as a list of GeoDataFrames (one per geometry in input).
            out_path (filepath): Optional filepath to save candidates. If flatten==False, out_path is ignored.

        Returns:
            gpd.GeoDataFrame: Holds candidate groups as MultiPoints in source CRS. (flatten=True)
            or
            list(gpd.GeoDataFrame): One GeoDataFrame of candidate points for each Linestring in source. (flatten=False)
        """

        if class_map:
            # set window for cropping geoms and clip geometries
            self.set_crop_window(class_map)
            gdf = gpd.clip(gdf, mask=self.crop_window, keep_geom_type=True)

        # Get list of groups of points
        all_sets = [self._generate_points(line) for line in gdf.geometry]

        if flatten:
            # Flatten list, convert to geodataframe
            all_candidates = gpd.GeoDataFrame(geometry=[point_group for line_set in all_sets for point_group in line_set], crs=gdf.crs)

            if out_path:
                if '.shp' not in os.path.splitext(out_path)[1]:
                    print("Warning (update_gdf): Saving candidate point GeoDataFrame to folder instead of file: '{}'.\nTo save as file, include the .shp extension.".format(out_path))

                all_candidates.to_file(out_path)

            return all_candidates
        else:
            # Return list of GeoDataFrames, one for each geometry in source annotation
            return [gpd.GeoDataFrame(geometry=point_set, crs=gdf.crs) for point_set in all_sets]


from django.urls import path

from . import views

app_name = 'olive'

urlpatterns = [
    path('', views.IndexView.as_view(), name='index'),
    path('shop/', views.ShopView.as_view(), name='shop'),
    # path('shop/<category>', views.ShopView.as_view(), name='shop_category'),
    path('<int:olive_id>/', views.DetailView.as_view(), name='detail'),
    path('order/<int:olive_id>/', views.OrderView.as_view(), name='order'),
    path('checkout/<int:olive_id>/', views.CustomerFormView.as_view(), name='checkout'),

]


import os
import pandas as pd
import numpy as np
import statsmodels.formula.api as smf
import matplotlib
import matplotlib.pyplot as pp

# Set pandas matplotlib style
matplotlib.rcParams.update(pd.tools.plotting.mpl_stylesheet)
matplotlib.rcParams.update({'font.family':'sans-serif'})

# 3rd degree polynomial
coeff = [0.4, -0.7, 0.3, 0]
sigma = 0.01

x = np.linspace(0, 1, 12)
y = np.polyval(coeff, x) + sigma*np.random.randn(len(x))

pp.figure(figsize=(8,4))#figsize=(3,3))
pp.subplot(121)

df = pd.DataFrame(data=dict(x=x, y=y))
for i in range(2, 11):
    df['x%d' % i] = df['x']**i
df.plot(x='x', y='y', style='.', label='data', ms=15)

m3 = smf.ols('y ~ x + x2 + x3', data=df).fit()
m6 = smf.ols('y ~ x + x2 + x3 + x4 + x5 + x6', data=df).fit()

df2 = pd.DataFrame(data=dict(x=np.linspace(0,1,1000)))
for i in range(2, 11):
    df2['x%d' % i] = df2['x']**i

df2['y_pred_3'] = m3.predict(df2)
df2['y_pred_6'] = m6.predict(df2)
df2.plot(x='x', y='y_pred_3', label='3rd degree')
df2.plot(x='x', y='y_pred_6', label='6th degree')
pp.xlabel('Model Inputs', size=16)
pp.ylabel('Predictions', size=16)
pp.legend(loc=1)
pp.gcf().subplots_adjust(bottom=0.15)

pp.subplot(122)

name = __file__ + '.png'
pp.savefig(name)
os.system('open %s' % name)


"""
#Faça um Programa que peça N notas de N alunos,
#calcule e armazene num vetor a média de cada aluno,
#imprima o somente alunos com média maior ou igual a 7.0 dizendo:  APROVADO
"""
alunos = 0
media = []
todas = []
qtd_alunos = int(input('Serão quantos alunos: '))
numero_notas = int(input('Qual o numero de notas coletadas: '))
#numero_notas = 2

while alunos < qtd_alunos: # OU 3
    print(20*'::')
    nota = [int(input('Nota: ')) for i in range(numero_notas)]
    soma=0.0
    for i in nota:
        soma += float(i)
    media_aluno = soma/numero_notas
    todas.append(nota)
    media.append(media_aluno)
    alunos += 1
else:
    [print(f'\nNotas do aluno {i+1}\n{todas[i]} média: {media[i]:.2f} - APROVADO\n') for i in range(alunos) if media[i] >= 7]


#!/usr/bin/env python
# coding: utf-8

# In[22]:

import numpy as np
import math
a = np.array([0,1,2,3,4,5])
b = np.array([42, 33, 14, 6, 4, 1])
fx = a*b
f = np.sum(b)
m = np.sum(fx)/f
print("so the mean of the data is {}".format(m))   #mean of the data
N = list()
prob = list()
for i in a:
    px = (((math.exp(-m))*(m**i))/(math.factorial(i)))
    prob.append(round(px,4))
    exp_f = round(px*f)
    N.append(exp_f)
print("******so the expected frequency and the associated probability according to the poisson distribution is *****")
print(a)
print(N)
print(prob)


# -*- coding: utf-8 -*-

class Awari(object):
    def solve(self, boxes_list):
        running = True
        solution = 'S'

        while running:
            print(boxes_list)

            nro_box = len(boxes_list)
            i = 0
            for rocks in boxes_list[::-1]:
                if nro_box == rocks:
                    i = rocks
                    boxes_list[nro_box - 1] = 0
                    break
                nro_box -= 1
            else:
                if sum(boxes_list) == 0:
                    solution = 'S'
                else:
                    solution = 'N'
                break

            boxes_remaining = boxes_list[:i - 1]
            boxes_remaining_len = len(boxes_remaining)
            for nro_box, rocks in enumerate(boxes_remaining, 1):
                if rocks != 0 and nro_box > rocks and i > 0:
                    boxes_list[nro_box - 1] += 1
                    i -= 1
                    boxes_remaining_len -= 1

            if boxes_remaining_len == len(boxes_remaining):
                break

        return solution


"""

"""
import logging
import copy

from django.conf import settings
from django.db import connection, connections

from core.pandajob.models import PandaJob
from core.pandajob.utils import identify_jobtype
from core.schedresource.utils import get_panda_queues
from core.libs.exlib import getPilotCounts
from core.libs.sqlsyntax import interval_to_sec

import core.constants as const

_logger = logging.getLogger('bigpandamon')

def prepare_job_summary_region(jsr_queues_dict, jsr_sites_dict, jsr_regions_dict, **kwargs):
    """
    Convert dict of region job summary to list
    :param jsr_queues_dict: dict of region job summary
    :param jsr_regions_dict: dict of queue job summary
    :return: list of region job summary, list of queue job summary
    """
    split_by = None
    if 'split_by' in kwargs and kwargs['split_by']:
        split_by = kwargs['split_by']
    jsr_queues_list = []
    jsr_sites_list = []
    jsr_regions_list = []
    for pq, params in jsr_queues_dict.items():
        for jt, resourcetypes in params['summary'].items():
            for rt, summary in resourcetypes.items():
                row = list()
                row.append(pq)
                row.append(params['pq_params']['pqtype'])
                row.append(params['pq_params']['region'])
                row.append(params['pq_params']['status'])
                row.append(jt)
                row.append(rt)
                row.append(params['pq_pilots']['count'])
                row.append(params['pq_pilots']['count_nojob'])
                row.append(summary['nwsubmitted'])
                row.append(summary['nwrunning'])
                row.append(summary['rcores'])
                row.append(sum([v for k, v in summary.items() if k in const.JOB_STATES]))
                if summary['failed'] + summary['finished'] > 0:
                    row.append(round(100.0*summary['failed']/(summary['failed'] + summary['finished']), 1))
                else:
                    row.append(0)
                for js in const.JOB_STATES:
                    row.append(summary[js])

                if split_by is None:
                    if jt == 'all' and rt == 'all':
                        jsr_queues_list.append(row)
                elif 'jobtype' in split_by and 'resourcetype' in split_by:
                    if jt != 'all' and rt != 'all':
                        jsr_queues_list.append(row)
                elif 'jobtype' in split_by and 'resourcetype' not in split_by:
                    if jt != 'all' and rt == 'all':
                        jsr_queues_list.append(row)
                elif 'jobtype' not in split_by and 'resourcetype' in split_by:
                    if jt == 'all' and rt != 'all':
                        jsr_queues_list.append(row)

    for site, jobtypes in jsr_sites_dict.items():
        for jt, resourcetypes in jobtypes.items():
            for rt, summary in resourcetypes.items():
                row = list()
                row.append(site)
                row.append(jt)
                row.append(rt)
                row.append(summary['nwsubmitted'])
                row.append(summary['nwrunning'])
                row.append(summary['rcores'])
                row.append(sum([v for k, v in summary.items() if k in const.JOB_STATES]))
                if summary['failed'] + summary['finished'] > 0:
                    row.append(round(100.0 * summary['failed'] / (summary['failed'] + summary['finished']), 1))
                else:
                    row.append(0)
                for js in const.JOB_STATES:
                    row.append(summary[js])

                if split_by is None:
                    if jt == 'all' and rt == 'all':
                        jsr_sites_list.append(row)
                elif 'jobtype' in split_by and 'resourcetype' in split_by:
                    if jt != 'all' and rt != 'all':
                        jsr_sites_list.append(row)
                elif 'jobtype' in split_by and 'resourcetype' not in split_by:
                    if jt != 'all' and rt == 'all':
                        jsr_sites_list.append(row)
                elif 'jobtype' not in split_by and 'resourcetype' in split_by:
                    if jt == 'all' and rt != 'all':
                        jsr_sites_list.append(row)

    for reg, jobtypes in jsr_regions_dict.items():
        for jt, resourcetypes in jobtypes.items():
            for rt, summary in resourcetypes.items():
                row = list()
                row.append(reg)
                row.append(jt)
                row.append(rt)
                row.append(summary['nwsubmitted'])
                row.append(summary['nwrunning'])
                row.append(summary['rcores'])
                row.append(sum([v for k, v in summary.items() if k in const.JOB_STATES]))
                if summary['failed'] + summary['finished'] > 0:
                    row.append(round(100.0 * summary['failed'] / (summary['failed'] + summary['finished']), 1))
                else:
                    row.append(0)
                for js in const.JOB_STATES:
                    row.append(summary[js])

                if split_by is None:
                    if jt == 'all' and rt == 'all':
                        jsr_regions_list.append(row)
                elif 'jobtype' in split_by and 'resourcetype' in split_by:
                    if jt != 'all' and rt != 'all':
                        jsr_regions_list.append(row)
                elif 'jobtype' in split_by and 'resourcetype' not in split_by:
                    if jt != 'all' and rt == 'all':
                        jsr_regions_list.append(row)
                elif 'jobtype' not in split_by and 'resourcetype' in split_by:
                    if jt == 'all' and rt != 'all':
                        jsr_regions_list.append(row)
    return jsr_queues_list, jsr_sites_list, jsr_regions_list

def get_job_summary_region(query, **kwargs):
    """
    :param query: dict of query params for jobs retrieving
    :return: dict of groupings
    """
    jsr_queues_dict = {}
    jsr_sites_dict = {}
    jsr_regions_dict = {}

    job_types = ['analy', 'prod']
    resource_types = ['SCORE', 'MCORE', 'SCORE_HIMEM', 'MCORE_HIMEM']
    worker_metrics = ['nwrunning', 'nwsubmitted']
    extra_metrics = copy.deepcopy(worker_metrics)
    extra_metrics.append('rcores')

    if 'extra' in kwargs and len(kwargs['extra']) > 1:
        extra = kwargs['extra']
    else:
        extra = '(1=1)'
    if 'region' in kwargs and kwargs['region'] != 'all':
        region = kwargs['region']
        regions_list = [region]
    else:
        region = 'all'
    if 'jobtype' in kwargs and kwargs['jobtype'] != 'all':
        jobtype = kwargs['jobtype']
        job_types = [kwargs['jobtype']]
    else:
        jobtype = 'all'
    if 'resourcetype' in kwargs and kwargs['resourcetype'] != 'all':
        resourcetype = kwargs['resourcetype']
        resource_types = [kwargs['resourcetype']]
    else:
        resourcetype = 'all'
    if 'split_by' in kwargs and kwargs['split_by']:
        split_by = kwargs['split_by']
    else:
        split_by = None
    if 'pqs_dict' in kwargs and kwargs['pqs_dict']:
        pqs_dict = kwargs['pqs_dict']
    else:
        # get PQ info
        pqs_dict = get_panda_queues()

    # filter out queues that do not match to provided jobtype (analy/prod)
    if jobtype != 'all':
        pqs_dict = {pqn: params for pqn, params in pqs_dict.items() if is_jobtype_match_queue(jobtype, params)}

    # filter out queues if there is computingsite in the query for jobs
    if 'computingsite' in query:
        pqs_dict = {pqn: params for pqn, params in pqs_dict.items() if pqn == query['computingsite']}
    elif 'computingsite__in' in query:
        pqs_dict = {pqn: params for pqn, params in pqs_dict.items() if pqn in query['computingsite__in']}

    sites_list = list(set([params['atlas_site'] for pq, params in pqs_dict.items() if 'atlas_site' in params]))
    regions_list = list(set([params['cloud'] for pq, params in pqs_dict.items() if 'cloud' in params]))

    # get job summary
    jsq = get_job_summary_split(query, extra=extra)

    # check if there is more values of resourcetype than 4 default ones, if yes -> add them to the list
    if len(jsq) > 0:
        job_resource_types_extra = list(set([row['resourcetype'] for row in jsq]) - set(resource_types))
        if None in job_resource_types_extra or '' in job_resource_types_extra:
            # replace None with 'Not specified'
            resource_types.append('Not specified')
            for row in jsq:
                if row['resourcetype'] is None or row['resourcetype'] == '':
                    row['resourcetype'] = 'Not specified'

        # adding extra job resourcetype to the list
        resource_types.extend([rt for rt in job_resource_types_extra if rt and rt != ''])

    # get workers info
    wsq = []
    if 'core.harvester' in settings.INSTALLED_APPS:
        from core.harvester.utils import get_workers_summary_split
        if 'computingsite__in' not in query:
            # put full list of compitingsites to use index in workers table
            query['computingsite__in'] = list(set([row['computingsite'] for row in jsq]))
        wsq = get_workers_summary_split(query)

    # get PanDA getJob, updateJob request counts
    psq_dict = {}
    if split_by is None and jobtype == 'all' and resourcetype == 'all':
        psq_dict = getPilotCounts('all')

    # create template structure for grouping by queue
    for pqn, params in pqs_dict.items():
        jsr_queues_dict[pqn] = {'pq_params': {}, 'pq_pilots': {},  'summary': {'all': {'all': {}}}}
        jsr_queues_dict[pqn]['pq_params']['pqtype'] = params['type'] if 'type' in params else '-'
        jsr_queues_dict[pqn]['pq_params']['region'] = params['cloud'] if 'cloud' in params else '-'
        jsr_queues_dict[pqn]['pq_params']['atlas_site'] = params['atlas_site'] if 'atlas_site' in params else '-'
        jsr_queues_dict[pqn]['pq_params']['status'] = params['status'] if 'status' in params else '-'
        jsr_queues_dict[pqn]['pq_pilots']['count'] = psq_dict[pqn]['count_abs'] if pqn in psq_dict else -1
        jsr_queues_dict[pqn]['pq_pilots']['count_nojob'] = psq_dict[pqn]['count_nojobabs'] if pqn in psq_dict else -1
        for jt in job_types:
            if is_jobtype_match_queue(jt, params):
                jsr_queues_dict[pqn]['summary'][jt] = {}
                for rt in resource_types:
                    jsr_queues_dict[pqn]['summary'][jt][rt] = {}
                    jsr_queues_dict[pqn]['summary'][jt]['all'] = {}
                    jsr_queues_dict[pqn]['summary']['all'][rt] = {}
                    for js in const.JOB_STATES:
                        jsr_queues_dict[pqn]['summary'][jt][rt][js] = 0
                        jsr_queues_dict[pqn]['summary'][jt]['all'][js] = 0
                        jsr_queues_dict[pqn]['summary']['all'][rt][js] = 0
                        jsr_queues_dict[pqn]['summary']['all']['all'][js] = 0

                    for em in extra_metrics:
                        jsr_queues_dict[pqn]['summary'][jt][rt][em] = 0
                        jsr_queues_dict[pqn]['summary'][jt]['all'][em] = 0
                        jsr_queues_dict[pqn]['summary']['all'][rt][em] = 0
                        jsr_queues_dict[pqn]['summary']['all']['all'][em] = 0

    # create template structure for grouping by region
    for r in regions_list:
        jsr_regions_dict[r] = {}
        for jt in job_types:
            jsr_regions_dict[r][jt] = {}
            jsr_regions_dict[r]['all'] = {}
            for rt in resource_types:
                jsr_regions_dict[r][jt][rt] = {}
                jsr_regions_dict[r][jt]['all'] = {}
                jsr_regions_dict[r]['all'][rt] = {}
                jsr_regions_dict[r]['all']['all'] = {}
                for js in const.JOB_STATES:
                    jsr_regions_dict[r][jt][rt][js] = 0
                    jsr_regions_dict[r][jt]['all'][js] = 0
                    jsr_regions_dict[r]['all'][rt][js] = 0
                    jsr_regions_dict[r]['all']['all'][js] = 0

                for em in extra_metrics:
                    jsr_regions_dict[r][jt][rt][em] = 0
                    jsr_regions_dict[r][jt]['all'][em] = 0
                    jsr_regions_dict[r]['all'][rt][em] = 0
                    jsr_regions_dict[r]['all']['all'][em] = 0

    # create template structure for grouping by site
    for s in sites_list:
        jsr_sites_dict[s] = {}
        for jt in job_types:
            jsr_sites_dict[s][jt] = {}
            jsr_sites_dict[s]['all'] = {}
            for rt in resource_types:
                jsr_sites_dict[s][jt][rt] = {}
                jsr_sites_dict[s][jt]['all'] = {}
                jsr_sites_dict[s]['all'][rt] = {}
                jsr_sites_dict[s]['all']['all'] = {}
                for js in const.JOB_STATES:
                    jsr_sites_dict[s][jt][rt][js] = 0
                    jsr_sites_dict[s][jt]['all'][js] = 0
                    jsr_sites_dict[s]['all'][rt][js] = 0
                    jsr_sites_dict[s]['all']['all'][js] = 0

                for em in extra_metrics:
                    jsr_sites_dict[s][jt][rt][em] = 0
                    jsr_sites_dict[s][jt]['all'][em] = 0
                    jsr_sites_dict[s]['all'][rt][em] = 0
                    jsr_sites_dict[s]['all']['all'][em] = 0

    # fill template with real values of job states counts
    for row in jsq:
        if row['computingsite'] in jsr_queues_dict and row['jobtype'] in jsr_queues_dict[row['computingsite']]['summary'] and row['resourcetype'] in resource_types and row['jobstatus'] in const.JOB_STATES and 'count' in row:
            jsr_queues_dict[row['computingsite']]['summary'][row['jobtype']][row['resourcetype']][row['jobstatus']] += int(row['count'])
            jsr_queues_dict[row['computingsite']]['summary']['all'][row['resourcetype']][row['jobstatus']] += int(row['count'])
            jsr_queues_dict[row['computingsite']]['summary'][row['jobtype']]['all'][row['jobstatus']] += int(row['count'])
            jsr_queues_dict[row['computingsite']]['summary']['all']['all'][row['jobstatus']] += int(row['count'])

            jsr_sites_dict[jsr_queues_dict[row['computingsite']]['pq_params']['atlas_site']][row['jobtype']][row['resourcetype']][row['jobstatus']] += int(row['count'])
            jsr_sites_dict[jsr_queues_dict[row['computingsite']]['pq_params']['atlas_site']]['all'][row['resourcetype']][row['jobstatus']] += int(row['count'])
            jsr_sites_dict[jsr_queues_dict[row['computingsite']]['pq_params']['atlas_site']][row['jobtype']]['all'][row['jobstatus']] += int(row['count'])
            jsr_sites_dict[jsr_queues_dict[row['computingsite']]['pq_params']['atlas_site']]['all']['all'][row['jobstatus']] += int(row['count'])

            jsr_regions_dict[jsr_queues_dict[row['computingsite']]['pq_params']['region']][row['jobtype']][row['resourcetype']][row['jobstatus']] += int(row['count'])
            jsr_regions_dict[jsr_queues_dict[row['computingsite']]['pq_params']['region']]['all'][row['resourcetype']][row['jobstatus']] += int(row['count'])
            jsr_regions_dict[jsr_queues_dict[row['computingsite']]['pq_params']['region']][row['jobtype']]['all'][row['jobstatus']] += int(row['count'])
            jsr_regions_dict[jsr_queues_dict[row['computingsite']]['pq_params']['region']]['all']['all'][row['jobstatus']] += int(row['count'])

            # fill sum of running cores
            if row['jobstatus'] == 'running':
                jsr_queues_dict[row['computingsite']]['summary'][row['jobtype']][row['resourcetype']]['rcores'] += int(row['rcores'])
                jsr_queues_dict[row['computingsite']]['summary']['all'][row['resourcetype']]['rcores'] += int(row['rcores'])
                jsr_queues_dict[row['computingsite']]['summary'][row['jobtype']]['all']['rcores'] += int(row['rcores'])
                jsr_queues_dict[row['computingsite']]['summary']['all']['all']['rcores'] += int(row['rcores'])

                jsr_sites_dict[jsr_queues_dict[row['computingsite']]['pq_params']['atlas_site']][row['jobtype']][row['resourcetype']]['rcores'] += int(row['rcores'])
                jsr_sites_dict[jsr_queues_dict[row['computingsite']]['pq_params']['atlas_site']]['all'][row['resourcetype']]['rcores'] += int(row['rcores'])
                jsr_sites_dict[jsr_queues_dict[row['computingsite']]['pq_params']['atlas_site']][row['jobtype']]['all']['rcores'] += int(row['rcores'])
                jsr_sites_dict[jsr_queues_dict[row['computingsite']]['pq_params']['atlas_site']]['all']['all']['rcores'] += int(row['rcores'])

                jsr_regions_dict[jsr_queues_dict[row['computingsite']]['pq_params']['region']][row['jobtype']][row['resourcetype']]['rcores'] += int(row['rcores'])
                jsr_regions_dict[jsr_queues_dict[row['computingsite']]['pq_params']['region']]['all'][row['resourcetype']]['rcores'] += int(row['rcores'])
                jsr_regions_dict[jsr_queues_dict[row['computingsite']]['pq_params']['region']][row['jobtype']]['all']['rcores'] += int(row['rcores'])
                jsr_regions_dict[jsr_queues_dict[row['computingsite']]['pq_params']['region']]['all']['all']['rcores'] += int(row['rcores'])

    # fill template with real values of n workers stats
    for row in wsq:
        if row['computingsite'] in jsr_queues_dict and row['jobtype'] in jsr_queues_dict[row['computingsite']]['summary'] and row['resourcetype'] in resource_types:
            for wm in worker_metrics:
                jsr_queues_dict[row['computingsite']]['summary'][row['jobtype']][row['resourcetype']][wm] += int(row[wm])
                jsr_queues_dict[row['computingsite']]['summary']['all'][row['resourcetype']][wm] += int(row[wm])
                jsr_queues_dict[row['computingsite']]['summary'][row['jobtype']]['all'][wm] += int(row[wm])
                jsr_queues_dict[row['computingsite']]['summary']['all']['all'][wm] += int(row[wm])

                jsr_sites_dict[jsr_queues_dict[row['computingsite']]['pq_params']['atlas_site']][row['jobtype']][row['resourcetype']][wm] += int(row[wm])
                jsr_sites_dict[jsr_queues_dict[row['computingsite']]['pq_params']['atlas_site']]['all'][row['resourcetype']][wm] += int(row[wm])
                jsr_sites_dict[jsr_queues_dict[row['computingsite']]['pq_params']['atlas_site']][row['jobtype']]['all'][wm] += int(row[wm])
                jsr_sites_dict[jsr_queues_dict[row['computingsite']]['pq_params']['atlas_site']]['all']['all'][wm] += int(row[wm])

                jsr_regions_dict[jsr_queues_dict[row['computingsite']]['pq_params']['region']][row['jobtype']][row['resourcetype']][wm] += int(row[wm])
                jsr_regions_dict[jsr_queues_dict[row['computingsite']]['pq_params']['region']]['all'][row['resourcetype']][wm] += int(row[wm])
                jsr_regions_dict[jsr_queues_dict[row['computingsite']]['pq_params']['region']][row['jobtype']]['all'][wm] += int(row[wm])
                jsr_regions_dict[jsr_queues_dict[row['computingsite']]['pq_params']['region']]['all']['all'][wm] += int(row[wm])

    return jsr_queues_dict, jsr_sites_dict, jsr_regions_dict

def get_job_summary_split(query, extra):
    """ Query the jobs summary """
    db = connections['default'].vendor
    fields = {field.attname: field.db_column for field in PandaJob._meta.get_fields()}
    extra_str = extra
    for qn, qvs in query.items():
        if '__in' in qn and qn.replace('__in', '') in fields:
            extra_str += " and (" + fields[qn.replace('__in', '')] + " in ("
            if len(qvs) > 0:
                for qv in qvs:
                    extra_str += "'" + str(qv) + "',"
                extra_str = extra_str[:-1]
            else:
                extra_str += "''"
            extra_str += "))"
        elif '__icontains' in qn or '__contains' in qn and qn.replace('__icontains', '').replace('__contains', '') in fields:
            extra_str += " and (" + fields[qn.replace('__icontains', '').replace('__contains', '')] + " LIKE '%%" + qvs + "%%')"
        elif '__startswith' in qn and qn.replace('__startswith', '') in fields:
            extra_str += " and (" + fields[qn.replace('__startswith', '')] + " LIKE '" + qvs + "%%')"
        elif '__endswith' in qn and qn.replace('__endswith', '') in fields:
            extra_str += " and (" + fields[qn.replace('__endswith', '')] + " LIKE '%%" + qvs + "')"
        elif qn in fields:
            extra_str += " and (" + fields[qn] + "= '" + str(qvs) + "' )"

    # get jobs groupings, the jobsactive4 table can keep failed analysis jobs for up to 7 days, so splitting the query
    query_raw = """
        select j.computingsite, j.resource_type as resourcetype, j.prodsourcelabel, j.transform, j.jobstatus,
            count(j.pandaid) as count, sum(j.rcores) as rcores, round(sum(j.walltime)) as walltime
        from  (
        select ja4.pandaid, ja4.resource_type, ja4.computingsite, ja4.prodsourcelabel, ja4.jobstatus, ja4.modificationtime, 0 as rcores,
            case when jobstatus in ('finished', 'failed') then {walltime_sec} else 0 end as walltime,
            case when transformation like 'http%' then 'run' when transformation like '%.py' then 'py' else 'unknown' end as transform
        from {db_scheme}.jobsarchived4 ja4  where modificationtime > TO_DATE('{date_from}', 'YYYY-MM-DD HH24:MI:SS') and {extra_str}
        union
        select jav4.pandaid, jav4.resource_type, jav4.computingsite, jav4.prodsourcelabel, jav4.jobstatus, jav4.modificationtime,
            case when jobstatus = 'running' then actualcorecount else 0 end as rcores, 0 as walltime,
            case when transformation like 'http%' then 'run' when transformation like '%.py' then 'py' else 'unknown' end as transform
        from {db_scheme}.jobsactive4 jav4 where modificationtime > TO_DATE('{date_from}', 'YYYY-MM-DD HH24:MI:SS') and
            jobstatus in ('failed', 'finished', 'closed', 'cancelled') and {extra_str}
        union
        select jav4.pandaid, jav4.resource_type, jav4.computingsite, jav4.prodsourcelabel, jav4.jobstatus, jav4.modificationtime,
            case when jobstatus = 'running' then actualcorecount else 0 end as rcores, 0 as walltime,
            case when transformation like 'http%' then 'run' when transformation like '%.py' then 'py' else 'unknown' end as transform
        from {db_scheme}.jobsactive4 jav4 where jobstatus not in ('failed', 'finished', 'closed', 'cancelled') and {extra_str}
        union
        select jw4.pandaid, jw4.resource_type, jw4.computingsite, jw4.prodsourcelabel, jw4.jobstatus, jw4.modificationtime, 0 as rcores, 0 as walltime,
            case when transformation like 'http%' then 'run' when transformation like '%.py' then 'py' else 'unknown' end as transform
        from {db_scheme}.jobswaiting4 jw4 where {extra_str}
        union
        select jd4.pandaid, jd4.resource_type, jd4.computingsite, jd4.prodsourcelabel, jd4.jobstatus, jd4.modificationtime, 0 as rcores, 0 as walltime,
            case when transformation like 'http%' then 'run' when transformation like '%.py' then 'py' else 'unknown' end as transform
        from {db_scheme}.jobsdefined4 jd4  where {extra_str}
        ) j
        GROUP BY j.computingsite, j.prodsourcelabel, j.transform, j.resource_type, j.jobstatus
        order by j.computingsite, j.prodsourcelabel, j.transform, j.resource_type, j.jobstatus
    """.format(
        date_from=query['modificationtime__castdate__range'][0],
        extra_str=extra_str,
        db_scheme=settings.DB_SCHEMA_PANDA,
        walltime_sec=interval_to_sec('endtime-starttime', db=db)
    )

    cur = connection.cursor()
    cur.execute(query_raw)
    job_summary_tuple = cur.fetchall()
    job_summary_header = ['computingsite', 'resourcetype', 'prodsourcelabel', 'transform', 'jobstatus', 'count', 'rcores', 'walltime']
    summary = [dict(zip(job_summary_header, row)) for row in job_summary_tuple]

    # Translate prodsourcelabel values to descriptive analy|prod job types
    summary = identify_jobtype(summary)

    return summary

def prettify_json_output(jsr_queues_dict, jsr_sites_dict, jsr_regions_dict, **kwargs):
    """
    Remove queues|regions with 0 jobs, add links to jobs page if requested
    :param jsr_queues_dict:
    :param jsr_regions_dict:
    :return:
    """
    region_summary = {}
    site_summary = {}
    queue_summary = {}

    is_add_link = False
    if 'extra' in kwargs and 'links' in kwargs['extra'] and kwargs['extra']['links'] is True:
        is_add_link = True
    if 'hours' in kwargs:
        hours = kwargs['hours']
    else:
        hours = 12

    base_url = 'https://bigpanda.cern.ch'
    jobs_path = '/jobs/?hours=' + str(hours)

    for rn, rdict in jsr_regions_dict.items():
        for jt, jtdict in rdict.items():
            for rt, rtdict in jtdict.items():
                if sum([rtdict[js] for js in const.JOB_STATES if js in rtdict]) > 0:
                    if rn not in region_summary:
                        region_summary[rn] = {}
                    if jt not in region_summary[rn]:
                        region_summary[rn][jt] = {}
                    if rt not in region_summary[rn][jt]:
                        region_summary[rn][jt][rt] = []
                    for js in const.JOB_STATES:
                        temp_dict = {
                            'jobstatus': js,
                            'count': rtdict[js]
                        }
                        if is_add_link:
                            temp_dict['job_link'] = base_url + jobs_path + '&region=' + rn + '&jobstatus=' + js
                            if jt != 'all':
                                temp_dict['job_link'] += '&jobtype=' + jt
                            if rt != 'all':
                                temp_dict['job_link'] += '&resourcetype=' + rt
                        region_summary[rn][jt][rt].append(temp_dict)

    for rn, rdict in jsr_sites_dict.items():
        for jt, jtdict in rdict.items():
            for rt, rtdict in jtdict.items():
                if sum([rtdict[js] for js in const.JOB_STATES if js in rtdict]) > 0:
                    if rn not in site_summary:
                        site_summary[rn] = {}
                    if jt not in site_summary[rn]:
                        site_summary[rn][jt] = {}
                    if rt not in site_summary[rn][jt]:
                        site_summary[rn][jt][rt] = []
                    for js in const.JOB_STATES:
                        temp_dict = {
                            'jobstatus': js,
                            'count': rtdict[js]
                        }
                        if is_add_link:
                            temp_dict['job_link'] = base_url + jobs_path + '&region=' + rn + '&jobstatus=' + js
                            if jt != 'all':
                                temp_dict['job_link'] += '&jobtype=' + jt
                            if rt != 'all':
                                temp_dict['job_link'] += '&resourcetype=' + rt
                        site_summary[rn][jt][rt].append(temp_dict)

    for qn, qdict in jsr_queues_dict.items():
        for jt, jtdict in qdict['summary'].items():
            for rt, rtdict in jtdict.items():
                if sum([rtdict[js] for js in const.JOB_STATES if js in rtdict]) > 0:
                    if qn not in queue_summary:
                        queue_summary[qn] = qdict['pq_params']
                        queue_summary[qn]['job_summary'] = {}
                    if jt not in queue_summary[qn]['job_summary']:
                        queue_summary[qn]['job_summary'][jt] = {}
                    if rt not in queue_summary[qn]['job_summary'][jt]:
                        queue_summary[qn]['job_summary'][jt][rt] = []
                    for js in const.JOB_STATES:
                        temp_dict = {
                            'jobstatus': js,
                            'count': rtdict[js]
                        }
                        if is_add_link:
                            temp_dict['job_link'] = base_url + jobs_path + '&computingsite=' + qn + '&jobstatus=' + js
                            if jt != 'all':
                                temp_dict['job_link'] += '&jobtype=' + jt
                            if rt != 'all':
                                temp_dict['job_link'] += '&resourcetype=' + rt
                        queue_summary[qn]['job_summary'][jt][rt].append(temp_dict)

    return queue_summary, site_summary, region_summary

def is_jobtype_match_queue(jobtype, pq_dict):
    """
    Check if analy/prod job can be run in a PanDA queue
    :param jobtype: str: analy or prod
    :param pq_dict: dict: PQ info
    :return: bool
    """
    pq_type = pq_dict['type'] if 'type' in pq_dict else 'unified'
    pq_jt_matrix = {
        'unified': ('prod', 'analy'),
        'production': ('prod',),
        'analysis': ('analy',),
        'special': ('prod',),
    }
    if pq_type in pq_jt_matrix and jobtype in pq_jt_matrix[pq_type]:
        return True

    return False


import xml.etree.ElementTree as ET

class Parser:
    def __init__(self, name):
        self.tree = ET.parse(name)
        self.iter_ = self.tree.iter()
        self.functions = {}
        self.bufer = {}

    def parse_childs(self,data):

        if data.getchildren():
            self.bufer[data.attrib['count']] = []
            self.parse_childs(self, data.getchildren())
        else:
            if data.tag=='integer':
                val=0
            elif data.tag=='string':
                val = ''
            elif data.tag=='double':
                val = 1.1
            if data.attrib['name'] in self.bufer:
                self.bufer[data.attrib['name']]=val

    def parse_it(self):
        for elem in self.iter_:
            inp={}
            out={}
            #print(elem.attrib)
            for fn in elem.getchildren():
                if fn.tag=='input':
                    self.bufer={}
                    #print(fn.getchildren())
                if fn.tag == 'output':
                    break
            if 'name' in elem.keys() and 'rpcf_' in elem.attrib['name']:
                self.functions[elem.attrib['name']] = {'id': elem.attrib['id'], 'input': inp,'output': out}
        #print(self.functions)

    def runner(self, attr, *args, **kwargs):
        '''метод, который будет имитировать все прочие методы'''
        function = attr
        if function in self.functions.keys():
            print(function, *args, **kwargs)
        else:
            print('No function found')

    def __getattr__(self,attr):
        def wrap(*args,**kwargs):
            return self.runner(attr,*args,**kwargs)
        return wrap

obj = Parser('api_53-004.xml')
obj.parse_it()
obj.rpcf_get_stats({'type':1235})


# 색종이 만들기(2630번)
###################################################################
    # 문제: https://www.acmicpc.net/problem/2630
    # 재귀, 분할정복
    # 역시 재귀는 나한테 어렵게 느껴진다
    # 원리는 알겠고, 쪼개야하는 것도 알겠는데 어느 순간에 뭘 기준으로 합치는지 잘 모르겠다.
    # 문제 원리는 단순하다. 분할 정복해서 구간을 나눈 뒤에 for 구문을 순회하면서 파란색 블럭을 센다.
    # 그 후에 파란색 블럭의 개수가 0개라면 그 구간만큼 흰색 종이의 수를 += 1하고
    # 만약 파란색 종이의 크기가 현재 구간의 너비와 같다면 파란색 종이의 수를 +=1한다.
###################################################################
import sys
sys.setrecursionlimit(int(1e5))
input = sys.stdin.readline

def recur(r, c, w):
    global b_cnt, w_cnt

    cnt = 0
    # 분할 정복하여 구간을 나눠서 파란색 색종이의 수를 센다
    for i in range(r, r + w):
        for j in range(c, c + w):
            if arr[i][j]:
                cnt += 1

    if not cnt:
        w_cnt += 1
    elif cnt == w * w:
        b_cnt += 1
    else:
        recur(r, c, w // 2)
        recur(r + w // 2, c, w // 2)
        recur(r, c + w // 2, w // 2)
        recur(r + w // 2, c + w // 2, w // 2)
    return

N = int(input())
arr = [list(map(int, input().split())) for _ in range(N)]

w_cnt, b_cnt = 0, 0

recur(0, 0, N)

print(w_cnt)
print(b_cnt)


import argparse
from pathlib import Path

from tqdm import tqdm

from processor.asr.yitu_asr_processor import yitu_asr_wrapper
from util.log_util import getLogger
from util.redis_util import getRedisClient
from util.util import mapLineCount

logger = getLogger('read_selenium_output')

from postprocess.line_processors import *

if __name__ == '__main__':
    parser = argparse.ArgumentParser()

    parser.add_argument('--input', required=True)
    parser.add_argument('--pattern')
    parser.add_argument('--postfix')

    args = parser.parse_args()

    input_path = Path(args.input)

    assert input_path.exists()

    if input_path.is_dir():
        assert args.pattern is not None
        logger.info(f'find files in {input_path}/{args.pattern}')
        files = [file for file in input_path.rglob(args.pattern)]
    else:
        files = [input_path]

    output_path = input_path.parent.joinpath(f'{input_path.stem}{args.postfix}')
    fw = output_path.open('w')
    logger.info(f'write file into {output_path}')

    if len(files) == 0:
        logger.warning(f'no file found, exit')
        exit(1)

    redis_cli = getRedisClient(db=0)

    for file in files:
        total_line_num = mapLineCount(str(file))
        fw = file.parent.joinpath(f'{file.name}_{args.postfix}').open('w')

        with file.open('rb') as fr:
            for lineno, line in tqdm(enumerate(fr, start=1), total=total_line_num):

                line = line.decode('utf-8', errors='ignore')
                line = line.strip()
                if line == '':
                    continue

                line_key = f"{file}:{lineno}"

                redis_line = redis_cli.get(line_key)
                if redis_line is not None:
                    continue

                res = yitu_asr_wrapper(line, line_key)

                fw.write(res + '\n')


t = input
p = print
r = range
n = int(t())
s1 = "I hate it"
s2 = "I love it"
s3 = "I hate that"
s4 = "I love that"
ans = [''] * n
for i in r(n -1):
    if i & 1:
        ans[i] = s4
    else:
        ans[i] = s3
p(' '.join(ans) + (s1 if n & 1 == 1 else s2))


# -*- coding: utf-8 -*-
"""
Created on Tue Mar  5 09:26:33 2019

@author: Tristan O'Hanlon
"""
import matplotlib.pyplot as plt
import math
import h5py
import os
os.chdir('E:/University/University/MSc/Models/climate-analysis/CCCM/reduced_datasets')
f = h5py.File('2010_CCCM_global_latitude.h5', 'r')

tcc = f['Cloud Fraction'][:]
tclw = f['Liquid Water Content'][:] # in g/m^3
tciw = f['Ice Water Content'][:] # in g/m^3

# Plot cloud fraction, liquid and ice water content vs latitude.
plt.figure()
fig, ax = plt.subplots()

#ax2 = ax1.twinx()
ax.plot(cccm_tclw_lat[:,0], cccm_tclw_lat[:,1], '-r', label='Liquid Water Content')
ax.plot(cccm_tciw_lat[:,0], cccm_tciw_lat[:,1], '--r', label='Ice Water Content')
ax.plot(cccm_tclw_av_lat[:,0], cccm_tclw_av_lat[:,1], '-b', label='Mean Liquid Water Content')
ax.plot(cccm_tciw_av_lat[:,0], cccm_tciw_av_lat[:,1], '--b', label='Mean Ice Water Content')

ax.legend(loc='lower center', bbox_to_anchor=(0.5, -0.3),
          ncol=4, fancybox=True, shadow=True);
#ax2.legend(loc='lower center', bbox_to_anchor=(0.5, -0.4),
#          ncol=4, fancybox=True, shadow=True);
ax.set_ylabel('Liquid and Ice Water Content', color='r')
ax.set_xlabel('Latitude')
#ax2.set_ylabel('Liquid and Ice Water Content ($gm^{-3}$)', color='r')

plt.title('Global Cloud Cover and Phase Content vs Latitude - CCCM 2010')

plt.grid(True)
plt.show()


from django.db import models
from stores.models import Store
from utils.image_upload_helper import upload_space_image_path
from utils.helpers import model_cleaner
from utils.helpers import autoslugFromUUID

@autoslugFromUUID()
class Space(models.Model):
    name = models.CharField(max_length=150)
    store = models.ForeignKey(Store, on_delete=models.CASCADE, related_name="store_spaces")
    slug = models.SlugField(unique=True, max_length=254)
    image_1 = models.ImageField(upload_to=upload_space_image_path, blank=True, null=True)
    image_1_reference = models.CharField(max_length=254, blank=True, null=True)
    image_1_comment = models.CharField(max_length=254, blank=True, null=True)
    image_2 = models.ImageField(upload_to=upload_space_image_path, blank=True, null=True)
    image_2_reference = models.CharField(max_length=254, blank=True, null=True)
    image_2_comment = models.CharField(max_length=254, blank=True, null=True)
    image_3 = models.ImageField(upload_to=upload_space_image_path, blank=True, null=True)
    image_3_reference = models.CharField(max_length=254, blank=True, null=True)
    image_3_comment = models.CharField(max_length=254, blank=True, null=True)
    image_4 = models.ImageField(upload_to=upload_space_image_path, blank=True, null=True)
    image_4_reference = models.CharField(max_length=254, blank=True, null=True)
    image_4_comment = models.CharField(max_length=254, blank=True, null=True)
    image_5 = models.ImageField(upload_to=upload_space_image_path, blank=True, null=True)
    image_5_reference = models.CharField(max_length=254, blank=True, null=True)
    image_5_comment = models.CharField(max_length=254, blank=True, null=True)
    equipment_details = models.TextField(blank=True, null=True)
    created_at = models.DateField(auto_now_add=True)
    updated_at = models.DateField(auto_now=True)

    class Meta:
        verbose_name = "Space"
        verbose_name_plural = "Spaces"
        ordering = ["-created_at"]

    def __str__(self):
        return self.name

    def clean(self, initialObject=None, requestObject=None):
        store_id = initialObject.store.id if initialObject else self.store.id if self.store else None
        qsFieldObjectList = [
            {
                "qs": self.__class__.objects.filter(name__iexact=self.name.lower(), store__id=store_id),
                "field": "name"
            }
        ]
        model_cleaner(selfObj=self, qsFieldObjectList=qsFieldObjectList, initialObject=initialObject)


"""empty message

Revision ID: 1a39e327a561
Revises: b6f61b196b53
Create Date: 2019-05-07 12:37:20.846345

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '1a39e327a561'
down_revision = 'b6f61b196b53'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('image', sa.Column('file', sa.String(length=100), nullable=True))
    op.add_column('image', sa.Column('uploaded_by', sa.Integer(), nullable=True))
    op.create_foreign_key(None, 'image', 'user', ['uploaded_by'], ['id'])
    op.drop_column('image', 'owner_id')
    op.drop_column('image', 'path')
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('image', sa.Column('path', sa.VARCHAR(length=255), nullable=True))
    op.add_column('image', sa.Column('owner_id', sa.INTEGER(), nullable=True))
    op.drop_constraint(None, 'image', type_='foreignkey')
    op.drop_column('image', 'uploaded_by')
    op.drop_column('image', 'file')
    # ### end Alembic commands ###


# сначала идут импорты из стандартных библиотек,
import random

# затем импорт библиотек установленных pip,
# import cv2

# затем локальные
import db

RATES = {
    'USD': random.randint(35, 120),
    'EUR': random.randint(40, 140),
}

def get_currency_rate(currency):
    return RATES.get(currency)

# чтобы можно было потестировать, и не запусалась  из __main__
if __name__ == '__main__':
    rate_1 = get_currency_rate('USD')
    rate_2 = get_currency_rate('EUR')
    db.write_to_db('USD', rate_1, '27/12/2022 23:10:15')
    print(rate_1, rate_2)


import json
import numpy as np
import torch
import torch.nn as nn
from torch.utils.data import DataLoader
from tqdm import tqdm
import click
from transformers import AutoProcessor, AutoModel
from vl_bench.data import Dataset_v2, get_xclip_collate_fn
from vl_bench.utils import process_path

MODELS = ("microsoft/xclip-base-patch32",)

@click.command()
@click.option(
    "-i", "--input-file", type=click.Path(exists=True, file_okay=True), required=True
)
@click.option(
    "-m",
    "--model-name",
    type=click.Choice(choices=MODELS),
    default=MODELS[0],
)
@click.option(
    "--batch-size",
    type=int,
    default=16,
)
@click.option(
    '--num-workers',
    type=int,
    default=5,
)
@click.option(
    "--device",
    type=str,
    default="cuda:0" if torch.cuda.is_available() else "cpu",
)
@click.option(
    "--quva-dir",
    type=click.Path(exists=True, dir_okay=True),
    required=False,
)
@click.option(
    "--something-something-dir",
    type=click.Path(exists=True, dir_okay=True),
    required=False,
)
@click.option(
    '--youtube-dir',
    type=click.Path(exists=True, dir_okay=True),
    required=False,
)
@click.option(
    '--star-dir',
    type=click.Path(exists=True, dir_okay=True),
    required=False,
)
@click.option(
    '-o', '--output-file',
    type=click.Path(file_okay=True),
    required=True,
)
@click.option(
    '--proficiency',
    is_flag=True,
)
@click.option("--mask-video", type=bool, required=True, default=False)
def main(
    input_file,
    model_name,
    batch_size,
    num_workers,
    device,
    quva_dir,
    something_something_dir,
    youtube_dir,
    star_dir,
    output_file,
    proficiency,
    mask_video,
):
    print(f"- running xclip on {input_file}")
    print(f"- output to {output_file}")
    # check video datasets' dirs
    assert quva_dir is not None \
        or something_something_dir is not None \
        or youtube_dir is not None
    if quva_dir is not None:
        quva_dir = process_path(quva_dir)
    if something_something_dir is not None:
        something_something_dir = process_path(something_something_dir)
    if youtube_dir is not None:
        youtube_dir = process_path(youtube_dir)
    if star_dir is not None:
        star_dir = process_path(star_dir)
    np.random.seed(0)

    # initialize model & processor
    model_name = "microsoft/xclip-base-patch32"
    processor = AutoProcessor.from_pretrained(model_name)
    model = AutoModel.from_pretrained(model_name).half().to(device)

    # read data
    data = Dataset_v2(
        input_file,
        quva_dir=quva_dir,
        something_something_dir=something_something_dir,
        youtube_dir=youtube_dir,
        star_dir=star_dir,
        proficiency=proficiency,
    )
    loader = DataLoader(
        data,
        batch_size=batch_size,
        shuffle=False,
        collate_fn=get_xclip_collate_fn(processor=processor),
        num_workers=num_workers,
        # pin_memory=False,
    )

    results = dict()
    for i, batch in enumerate(tqdm(loader)):
        inputs = batch['inputs'].to(device)
        num_batch_texts = batch['num_texts']
        batch_size = len(num_batch_texts)

        with torch.no_grad():
            outputs = model(**inputs)
        logits = outputs.logits_per_video.cpu()

        offset = 0
        for i in range(batch_size):
            num_texts = num_batch_texts[i]
            start = offset
            end = offset + num_texts
            scores = logits[i, start:end].flatten().tolist()
            key = batch['item_ids'][i]
            results[key] = {'scores': scores}
            offset += num_texts

    with open(process_path(output_file), "w") as f:
        json.dump(results, f, indent=4)

if __name__ == "__main__":
    main()


#!/usr/bin/python
## coding:utf-8

#import sys
#sys.path.append("..")
#import main
#
#print("main.sqrt(72) is %f"%main.square_root(72))

import logging, mymodule

logging.basicConfig()
logger = logging.getLogger("MyApp")

logger.setLevel(logging.DEBUG)
logger.info("Start my app")
logger = logging.getLogger("Another App")
logger.info("another.. my app")
logger.info("Start my app")
logger = logging.getLogger("MyApp")
logger.info("another.. my app")
logger.info("Start my app")

logger = logging.getLogger("Another App")

try:
    mymodule.doIt()
except Exception as e:
    logger.exception("There was a problem")

logger.info("Ending my app")


from odoo import api, models, tools
from odoo.http import request

class Website(models.Model):
    _inherit = "website"

    @api.model
    def get_available_website_user_group_ids(self):
        return [
            self.env.ref("base.group_user").id,
            self.env.ref("website_user_types.group_b2c").id,
            self.env.ref("website_user_types.group_b2b").id,
            self.env.ref("base.group_public").id,
        ]

    def get_website_user_group_cache_key(self):
        user_group_ids = self.env.user.groups_id.ids
        for group_id in self.get_available_website_user_group_ids():
            if group_id in user_group_ids:
                return f"website_user_types_group_{group_id}"

        return "website_user_types_group_{}".format(
            self.env.ref("base.group_public").id
        )

    @tools.ormcache('self.env.uid')
    def get_website_user_group_classes(self):
        if self.user_has_groups("account.group_show_line_subtotals_tax_included"):
            wut_class = "wut_tax_included"
        else:
            wut_class = "wut_tax_excluded"

        if self.user_has_groups("website_user_types.group_b2b"):
            return "{} wut_group_b2b".format(wut_class)
        if self.user_has_groups("website_user_types.group_b2c"):
            return "{} wut_group_b2c".format(wut_class)
        return wut_class

    def sale_get_order(
        self,
        force_create=False,
        update_pricelist=False,
    ):
        sale_order = super().sale_get_order(
            force_create=force_create, update_pricelist=update_pricelist
        )
        if update_pricelist or force_create:
            partner = sale_order.partner_id
            if partner.business_relationship_id.update_prices:
                pricelist = self._get_default_business_relationship_pricelist(
                    sale_order
                )
                fiscal_position = self.env[
                    'account.fiscal.position'
                ].sudo()._get_fiscal_position(partner, sale_order.partner_shipping_id)

                if (
                    pricelist != sale_order.pricelist_id
                    or fiscal_position != sale_order.fiscal_position_id
                ):
                    if pricelist != sale_order.pricelist_id:
                        request.session['website_sale_current_pl'] = pricelist.id

                    sale_order.write({
                        'pricelist_id': pricelist.id,
                        'fiscal_position_id': fiscal_position.id,
                    })
                    sale_order._recompute_prices()

        return sale_order

    def _get_default_business_relationship_pricelist(self, order):
        partner = order.partner_id

        if partner.business_relationship_id.update_pricelist_by == "shipping":
            partner = order.partner_shipping_id

        return partner.property_product_pricelist

    @api.model
    def website_domain(self, website_id=False):
        # there is no perfect solution, either fully override _serve_page, or
        # use with_group_ids context for website but not view to keep compatibility high
        domain = super().website_domain(website_id=website_id)
        group_ids = self.env.context.get("with_group_ids", False)
        if group_ids:
            domain = domain + [
                ("|"),
                ("group_ids", "=", False),
                ("group_ids", "in", group_ids),
            ]

        return domain

    def sale_product_domain(self):
        domain = super().sale_product_domain()

        if self.user_has_groups("website.group_website_designer"):
            return domain

        # A user having both groups cannot be configured with business relationships,
        # but you can do so in the technical admin user settings
        if self.user_has_groups(
            "website_user_types.group_b2b"
        ) and self.user_has_groups("website_user_types.group_b2c"):
            domain = domain + [
                ("|"),
                ("visible_group_b2c", "=", True),
                ("visible_group_b2b", "=", True),
            ]
        else:
            if self.user_has_groups("website_user_types.group_b2b"):
                domain.append(
                    ("visible_group_b2b", "=", True),
                )
            else:
                domain.append(
                    ("visible_group_b2c", "=", True),
                )

        return domain

class View(models.Model):
    _inherit = "ir.ui.view"
    _name = "ir.ui.view"

    @api.model
    @tools.ormcache_context(
        "self.env.uid", "self.env.su", "xml_id", keys=("website_id",)
    )
    def get_view_id(self, xml_id):
        self = self.with_context(with_group_ids=False)
        return super().get_view_id(xml_id)

    @api.model
    def _get_inheriting_views_arch_domain(self, model):
        self = self.with_context(with_group_ids=False)
        return super()._get_inheriting_views_arch_domain(model)


from keyboards.user_keyboards.subscribe_channels import create_subscribe
from utils.json_worker.channels import get_channels, del_channel
from utils.misc.markreplace import markdowned

async def action_cd(callback):
    try:
        data = await get_channels()
        channel_id = callback.data.split('?')[1]
        channel_title = data[channel_id]['title']
        await del_channel(channel_id)
        await callback.message.answer(await markdowned(f'Канал *{channel_title}* успешно удален!'),
                                      reply_markup=await create_subscribe(url=False, cancel=True),
                                      parse_mode="MarkdownV2")

    except Exception as e:
        await callback.message.reply('ERROR!\n' + str(e))


#!/usr/bin/env python3

"""Advent of Code 2021 Day 25 - Sea Cucumber"""

with open('inputs/day_25.txt', 'r') as aoc_input:
    lines = [line.strip() for line in aoc_input.readlines()]

east_cucumbers = set()
south_cucumbers = set()
for y, row in enumerate(lines):
    for x, value in enumerate(row):
        if value == '>':
            east_cucumbers.add((x, y))
        elif value == 'v':
            south_cucumbers.add((x, y))

steps = 0
while True:
    new_east = set()
    new_south = set()

    for coords in east_cucumbers:

        x, y = coords
        forward_x = (x + 1) % len(lines[0])
        forward = (forward_x, y)

        if forward not in east_cucumbers and forward not in south_cucumbers:
            new_east.add(forward)
        else:
            new_east.add(coords)

    for coords in south_cucumbers:

        x, y = coords
        forward_y = (y + 1) % len(lines)
        forward = (x, forward_y)

        if forward not in new_east and forward not in south_cucumbers:
            new_south.add(forward)
        else:
            new_south.add(coords)

    steps += 1
    if new_east == east_cucumbers and new_south == south_cucumbers:
        break

    east_cucumbers = new_east
    south_cucumbers = new_south

# Answer One
print(f'First step on which no sea cucumbers move: {steps}')


#! /usr/bin/env python3
"""This demo uses Aboleth for approximate Gaussian process regression."""
import logging

import numpy as np
import bokeh.plotting as bk
import bokeh.palettes as bp
import tensorflow as tf
# from sklearn.gaussian_process.kernels import Matern as kern
from sklearn.gaussian_process.kernels import RBF as kern

import aboleth as ab
from aboleth.datasets import gp_draws

# Set up a python logger so we can see the output of MonitoredTrainingSession
logger = logging.getLogger()
logger.setLevel(logging.INFO)

# Set up a consistent random seed in Aboleth so we get repeatable, but random
# results
RSEED = 666
ab.set_hyperseed(RSEED)

# Data settings
N = 1000  # Number of training points to generate
Ns = 400  # Number of testing points to generate
kernel = kern(length_scale=0.5)  # Kernel to use for making a random GP draw
true_noise = 0.1  # Add noise to the GP draws, to make things a little harder

# Model settings
n_samples = 5  # Number of random samples to get from an Aboleth net
p_samples = 30  # Number of samples for prediction
n_epochs = 200  # how many times to see the data for training
batch_size = 10  # mini batch size for stochastric gradients
config = tf.ConfigProto(device_count={'GPU': 0})  # Use GPU? 0 is no

# Model initialisation
NOISE = 1.  # Likelihood st. dev. initialisation, and learning

# Random Fourier Features
kern = ab.RBF(learn_lenscale=True)  # keep the length scale positive

# Variational Fourier Features -- length-scale setting here is the "prior", we
# can choose to optimise this or not
# lenscale = 1.
# kern = ab.RBFVariational(lenscale=lenscale)  # This is VAR-FIXED kernel from
# Cutjar et. al. 2017

# This is how we make the "latent function" of a Gaussian process, here
# n_features controls how many random basis functions we use in the
# approximation. The more of these, the more accurate, but more costly
# computationally. "full" indicates we want a full-covariance matrix Gaussian
# posterior of the model weights. This is optional, but it does greatly improve
# the model uncertainty away from the data.
n_samples_ = tf.placeholder(tf.int32)
net = (
    ab.InputLayer(name="X", n_samples=n_samples_) >>
    ab.RandomFourier(n_features=200, kernel=kern) >>
    ab.DenseVariational(output_dim=1, learn_prior=True, full=True)
)

def main():
    """Run the demo."""
    n_iters = int(round(n_epochs * N / batch_size))
    print("Iterations = {}".format(n_iters))

    # Get training and testing data
    Xr, Yr, Xs, Ys = gp_draws(N, Ns, kern=kernel, noise=true_noise)

    # Prediction points
    Xq = np.linspace(-20, 20, Ns).astype(np.float32)[:, np.newaxis]
    Yq = np.linspace(-4, 4, Ns).astype(np.float32)[:, np.newaxis]

    # Set up the probability image query points
    Xi, Yi = np.meshgrid(Xq, Yq)
    Xi = Xi.astype(np.float32).reshape(-1, 1)
    Yi = Yi.astype(np.float32).reshape(-1, 1)

    _, D = Xr.shape

    # Name the "data" parts of the graph
    with tf.name_scope("Input"):
        # This function will make a TensorFlow queue for shuffling and batching
        # the data, and will run through n_epochs of the data.
        Xb, Yb = batch_training(Xr, Yr, n_epochs=n_epochs,
                                batch_size=batch_size)
        X_ = tf.placeholder_with_default(Xb, shape=(None, D))
        Y_ = tf.placeholder_with_default(Yb, shape=(None, 1))

    # This is where we build the actual GP model
    with tf.name_scope("Deepnet"):
        phi, kl = net(X=X_)
        noise = ab.pos_variable(NOISE)
        ll = tf.distributions.Normal(loc=phi, scale=noise).log_prob(Y_)
        loss = ab.elbo(ll, kl, N)

    # Set up the training graph
    with tf.name_scope("Train"):
        optimizer = tf.train.AdamOptimizer()
        global_step = tf.train.create_global_step()
        train = optimizer.minimize(loss, global_step=global_step)

    # This is used for building the predictive density image
    with tf.name_scope("Predict"):
        logprob = ab.sample_mean(ll)

    # Logging learning progress
    log = tf.train.LoggingTensorHook(
        {'step': global_step, 'loss': loss},
        every_n_iter=1000
    )

    # This is the main training "loop"
    with tf.train.MonitoredTrainingSession(
            config=config,
            save_summaries_steps=None,
            save_checkpoint_secs=None,
            hooks=[log]
    ) as sess:
        try:
            while not sess.should_stop():
                sess.run(train, feed_dict={n_samples_: n_samples})
        except tf.errors.OutOfRangeError:
            print('Input queues have been exhausted!')
            pass

        # Prediction, the [[None]] is to stop the default placeholder queue
        Ey = sess.run(phi,
                      feed_dict={X_: Xq, Y_: [[None]], n_samples_: p_samples})
        logPY = sess.run(logprob,
                         feed_dict={Y_: Yi, X_: Xi, n_samples_: p_samples})

    Eymean = Ey.mean(axis=0)  # Average samples to get mean predicted funtion
    Py = np.exp(logPY.reshape(Ns, Ns))  # Turn log-prob into prob

    # Plot
    im_min = np.amin(Py)
    im_size = np.amax(Py) - im_min
    img = (Py - im_min) / im_size
    f = bk.figure(tools='pan,box_zoom,reset', sizing_mode='stretch_both')
    f.image(image=[img], x=-20., y=-4., dw=40., dh=8,
            palette=bp.Plasma256)
    f.circle(Xr.flatten(), Yr.flatten(), fill_color='blue', legend='Training')
    f.line(Xs.flatten(), Ys.flatten(), line_color='blue', legend='Truth')
    for y in Ey:
        f.line(Xq.flatten(), y.flatten(), line_color='red', legend='Samples',
               alpha=0.2)
    f.line(Xq.flatten(), Eymean.flatten(), line_color='green', legend='Mean')
    bk.show(f)

def batch_training(X, Y, batch_size, n_epochs):
    """Batch training queue convenience function."""
    data_tr = tf.data.Dataset.from_tensor_slices({'X': X, 'Y': Y}) \
        .shuffle(buffer_size=1000, seed=RSEED) \
        .repeat(n_epochs) \
        .batch(batch_size)
    data = data_tr.make_one_shot_iterator().get_next()
    return data['X'], data['Y']

if __name__ == "__main__":
    main()


import os
import yaml
import time
import shutil
import torch
import random
import argparse
import numpy as np
from tensorboardX import SummaryWriter
from torch.utils import data
from models import get_model
from loader import get_loader
from optimizers import get_optimizer
from loss import get_loss_function
from utils import save_checkpoint, AverageMeter, accuracy, get_logger

def train(cfg, writer, logger, logdir):

    # Setup seeds
    if cfg['training']['seed'] is not None:
        seed = cfg['training']['seed']
        logger.info("Using seed {}".format(seed))
        torch.manual_seed(cfg.get("seed", cfg['training']['seed']))
        torch.cuda.manual_seed(cfg.get("seed", cfg['training']['seed']))
        np.random.seed(cfg.get("seed", cfg['training']['seed']))
        random.seed(cfg.get("seed", cfg['training']['seed']))

    # Setup device
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

    # Setup Dataloader
    t_loader = get_loader(cfg,'train')
    v_loader = get_loader(cfg,'val')

    trainloader = data.DataLoader(
        t_loader,
        batch_size=cfg["training"]["batch_size"],
        num_workers=cfg["training"]["n_workers"],
        shuffle=True,
    )

    valloader = data.DataLoader(
        v_loader,
        batch_size=cfg["training"]["batch_size"],
        num_workers=cfg["training"]["n_workers"]
    )
    n_classes = cfg['data']['num_classes']

    # Setup Model
    model_cfg = {}
    for item in cfg["model"]:
        if cfg["model"][item] is not None:
            model_cfg[item] = cfg["model"][item]
    model = get_model(**model_cfg, num_classes=n_classes, device=device).to(device)
    model = torch.nn.DataParallel(model, device_ids=range(torch.cuda.device_count()))

    # Setup optimizer, lr_scheduler and loss function
    optimizer_cls = get_optimizer(cfg["training"]["optimizer"])
    optimizer_params = {k: v for k, v in cfg["training"]["optimizer"].items() if k != "name"}
    optimizer = optimizer_cls(model.parameters(), **optimizer_params)
    logger.info("Using optimizer {}".format(optimizer))
    loss_fn = get_loss_function(cfg["training"]["loss"])
    logger.info("Using loss {}".format(loss_fn))

    if cfg['training']['scheduler'] == 'cos':
        scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=cfg['training']['train_epoch'])
    elif cfg['training']['scheduler'] == 'wide':
        scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, [60, 120, 160], 0.2)

    start_epoch = 0
    best_acc1 = -100.0

    # Resume pre-trained model
    if cfg["training"]["resume"] is not None:
        if os.path.isfile(cfg["training"]["resume"]):
            logger.info(
                "Loading model and optimizer from checkpoint '{}'".format(cfg["training"]["resume"])
            )
            checkpoint = torch.load(cfg["training"]["resume"])
            model.load_state_dict(checkpoint["state_dict"])
            optimizer.load_state_dict(checkpoint["optimizer"])
            best_acc1 = checkpoint["best_acc1"]
            start_epoch = checkpoint["epoch"]
            logger.info(
                "Loaded checkpoint '{}' (iter {})".format(
                    cfg["training"]["resume"], checkpoint["epoch"]
                )
            )
        else:
            logger.info("No checkpoint found at '{}'".format(cfg["training"]["resume"]))

  #================================ Training ==========================================
    for epoch in range(start_epoch, cfg['training']['train_epoch']):

        # Initilize trianing statistics
        batch_time = AverageMeter('Time', ':6.3f')
        data_time = AverageMeter('Data', ':6.3f')
        losses = AverageMeter('Loss', ':.4e')
        top1 = AverageMeter('Acc@1', ':6.2f')
        top5 = AverageMeter('Acc@5', ':6.2f')

        end = time.time()
        model.train()
        for i,(image, target) in enumerate(trainloader):
            data_time.update(time.time() - end)
            image = image.to(device)
            target = target.to(device)
            logit,_,_,_= model(image)
            loss = loss_fn(logit, target)
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

            # Measure accuracy and record loss
            acc1, acc5 = accuracy(logit, target, topk=(1, 5))
            losses.update(loss.item(), image.size(0))
            top1.update(acc1[0], image.size(0))
            top5.update(acc5[0], image.size(0))

            # Measure elapsed time
            batch_time.update(time.time() - end)
            end = time.time()

            if i % cfg["training"]["print_interval"] == 0:
                output = ('Epoch: [{0}][{1}/{2}], lr: {lr:.5f}\t'
                        'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                        'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
                        'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
                        'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
                        'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
                    epoch, i, len(trainloader), batch_time=batch_time,
                    data_time=data_time, loss=losses, top1=top1, top5=top5, lr=optimizer.param_groups[-1]['lr']))  # TODO
                print(output)
                logger.info(output + '\n')

        # Write stats to tensorboard
        writer.add_scalar('train/loss', losses.avg, epoch)
        writer.add_scalar('train/acc_top1', top1.avg, epoch)
        writer.add_scalar('train/acc_top5', top5.avg, epoch)
        writer.add_scalar('lr', optimizer.param_groups[-1]['lr'], epoch)

        # Update scheduler for learning rate decay
        scheduler.step()
        ##================== Evaluation ============================

        model.eval()
        with torch.no_grad():
            eval_top1 = AverageMeter('Acc@1', ':6.2f')
            eval_top5 = AverageMeter('Acc@5', ':6.2f')
            val_losses = AverageMeter('Loss', ':.4e')

            for i, (image, target) in enumerate(valloader):
                image = image.to(device)
                target = target.to(device)
                logit,_,_,_ = model(image)
                loss = loss_fn(logit, target)

                acc1, acc5 = accuracy(logit, target, topk=(1, 5))
                eval_top1.update(acc1[0], image.size(0))
                eval_top5.update(acc5[0], image.size(0))
                val_losses.update(loss.item(), image.size(0))

                if i % cfg["training"]["print_interval"] == 0:
                    output = ('Test: [{0}][{1}/{2}], lr: {lr:.5f}\t'
                            'Prec@1 {top1.avg:.3f}\t'
                            'Prec@5 {top5.avg:.3f}'.format(
                        epoch, i, len(valloader), top1=eval_top1, top5=eval_top5, lr=optimizer.param_groups[-1]['lr']))  # TODO
                    print(output)

                    output = ('validation Results: Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f}'
                            .format(top1=eval_top1, top5=eval_top5))
                    logger.info(output + '\n')

                writer.add_scalar('/loss', val_losses.avg, epoch)
                writer.add_scalar('/acc_top1', eval_top1.avg, epoch)
                writer.add_scalar('/acc_top5', eval_top5.avg, epoch)

            is_best = eval_top1.avg > best_acc1
            best_acc1 = max(eval_top1.avg, best_acc1)

        output_best = 'Best Prec@1: %.3f' % (best_acc1)
        logger.info(output_best + '\n')
        print(output_best)

        if (epoch+1) %200 == 0:
            filename = '{}/ckpt.{}.pth.tar'.format(logdir,epoch)
            torch.save({
                'epoch': epoch,
                'arch': cfg["model"]["arch"],
                'state_dict': model.state_dict(),
                'best_acc1': best_acc1,
                'optimizer' : optimizer.state_dict(),
            }, filename)

        save_checkpoint(logdir, {
            'epoch': epoch,
            'arch': cfg["model"]["arch"],
            'state_dict': model.state_dict(),
            'best_acc1': best_acc1,
            'optimizer' : optimizer.state_dict(),
        }, is_best)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="config")
    parser.add_argument(
        "--config",
        nargs="?",
        type=str,
        default="./configs/resnet18/wide.yaml",
        help="Configuration file to use",
    )

    args = parser.parse_args()

    with open(args.config) as fp:
        cfg = yaml.safe_load(fp)
        logdir = os.path.join("runs",cfg["data"]["name"],cfg["model"]["arch"],cfg['id'])
        if not os.path.exists(logdir):
            os.makedirs(logdir, exist_ok=True)
        print("RUNDIR: {}".format(logdir))
        shutil.copy(args.config, logdir)
        writer = SummaryWriter(logdir)
        logger = get_logger(logdir)
        logger.info("Let the games begin")
        train(cfg, writer, logger, logdir)