from __future__ import annotations

from typing import Any
from typing import Optional

from . import ext
from .._typing import _OnConflictConstraintT
from .._typing import _OnConflictIndexElementsT
from .._typing import _OnConflictIndexWhereT
from .._typing import _OnConflictSetT
from .._typing import _OnConflictWhereT
from ... import util
from ...sql import coercions
from ...sql import roles
from ...sql import schema
from ...sql._typing import _DMLTableArgument
from ...sql.base import _exclusive_against
from ...sql.base import _generative
from ...sql.base import ColumnCollection
from ...sql.base import ReadOnlyColumnCollection
from ...sql.dml import Insert as StandardInsert
from ...sql.elements import ClauseElement
from ...sql.elements import KeyedColumnElement
from ...sql.expression import alias
from ...util.typing import Self

__all__ = ("Insert", "insert")

def insert(table: _DMLTableArgument) -> Insert:
    """Construct a PostgreSQL-specific variant :class:`_postgresql.Insert`
    construct.

    .. container:: inherited_member

        The :func:`sqlalchemy.dialects.postgresql.insert` function creates
        a :class:`sqlalchemy.dialects.postgresql.Insert`.  This class is based
        on the dialect-agnostic :class:`_sql.Insert` construct which may
        be constructed using the :func:`_sql.insert` function in
        SQLAlchemy Core.

    The :class:`_postgresql.Insert` construct includes additional methods
    :meth:`_postgresql.Insert.on_conflict_do_update`,
    :meth:`_postgresql.Insert.on_conflict_do_nothing`.

    """
    return Insert(table)

class Insert(StandardInsert):
    """PostgreSQL-specific implementation of INSERT.

    Adds methods for PG-specific syntaxes such as ON CONFLICT.

    The :class:`_postgresql.Insert` object is created using the
    :func:`sqlalchemy.dialects.postgresql.insert` function.

    """

    stringify_dialect = "postgresql"
    inherit_cache = False

    @util.memoized_property
    def excluded(
        self,
    ) -> ReadOnlyColumnCollection[str, KeyedColumnElement[Any]]:
        """Provide the ``excluded`` namespace for an ON CONFLICT statement

        PG's ON CONFLICT clause allows reference to the row that would
        be inserted, known as ``excluded``.  This attribute provides
        all columns in this row to be referenceable.

        .. tip::  The :attr:`_postgresql.Insert.excluded` attribute is an
            instance of :class:`_expression.ColumnCollection`, which provides
            an interface the same as that of the :attr:`_schema.Table.c`
            collection described at :ref:`metadata_tables_and_columns`.
            With this collection, ordinary names are accessible like attributes
            (e.g. ``stmt.excluded.some_column``), but special names and
            dictionary method names should be accessed using indexed access,
            such as ``stmt.excluded["column name"]`` or
            ``stmt.excluded["values"]``.   See the docstring for
            :class:`_expression.ColumnCollection` for further examples.

        .. seealso::

            :ref:`postgresql_insert_on_conflict` - example of how
            to use :attr:`_expression.Insert.excluded`

        """
        return alias(self.table, name="excluded").columns

    _on_conflict_exclusive = _exclusive_against(
        "_post_values_clause",
        msgs={
            "_post_values_clause": "This Insert construct already has "
            "an ON CONFLICT clause established"
        },
    )

    @_generative
    @_on_conflict_exclusive
    def on_conflict_do_update(
        self,
        constraint: _OnConflictConstraintT = None,
        index_elements: _OnConflictIndexElementsT = None,
        index_where: _OnConflictIndexWhereT = None,
        set_: _OnConflictSetT = None,
        where: _OnConflictWhereT = None,
    ) -> Self:
        r"""
        Specifies a DO UPDATE SET action for ON CONFLICT clause.

        Either the ``constraint`` or ``index_elements`` argument is
        required, but only one of these can be specified.

        :param constraint:
         The name of a unique or exclusion constraint on the table,
         or the constraint object itself if it has a .name attribute.

        :param index_elements:
         A sequence consisting of string column names, :class:`_schema.Column`
         objects, or other column expression objects that will be used
         to infer a target index.

        :param index_where:
         Additional WHERE criterion that can be used to infer a
         conditional target index.

        :param set\_:
         A dictionary or other mapping object
         where the keys are either names of columns in the target table,
         or :class:`_schema.Column` objects or other ORM-mapped columns
         matching that of the target table, and expressions or literals
         as values, specifying the ``SET`` actions to take.

         .. versionadded:: 1.4 The
            :paramref:`_postgresql.Insert.on_conflict_do_update.set_`
            parameter supports :class:`_schema.Column` objects from the target
            :class:`_schema.Table` as keys.

         .. warning:: This dictionary does **not** take into account
            Python-specified default UPDATE values or generation functions,
            e.g. those specified using :paramref:`_schema.Column.onupdate`.
            These values will not be exercised for an ON CONFLICT style of
            UPDATE, unless they are manually specified in the
            :paramref:`.Insert.on_conflict_do_update.set_` dictionary.

        :param where:
         Optional argument. If present, can be a literal SQL
         string or an acceptable expression for a ``WHERE`` clause
         that restricts the rows affected by ``DO UPDATE SET``. Rows
         not meeting the ``WHERE`` condition will not be updated
         (effectively a ``DO NOTHING`` for those rows).

        .. seealso::

            :ref:`postgresql_insert_on_conflict`

        """
        self._post_values_clause = OnConflictDoUpdate(
            constraint, index_elements, index_where, set_, where
        )
        return self

    @_generative
    @_on_conflict_exclusive
    def on_conflict_do_nothing(
        self,
        constraint: _OnConflictConstraintT = None,
        index_elements: _OnConflictIndexElementsT = None,
        index_where: _OnConflictIndexWhereT = None,
    ) -> Self:
        """
        Specifies a DO NOTHING action for ON CONFLICT clause.

        The ``constraint`` and ``index_elements`` arguments
        are optional, but only one of these can be specified.

        :param constraint:
         The name of a unique or exclusion constraint on the table,
         or the constraint object itself if it has a .name attribute.

        :param index_elements:
         A sequence consisting of string column names, :class:`_schema.Column`
         objects, or other column expression objects that will be used
         to infer a target index.

        :param index_where:
         Additional WHERE criterion that can be used to infer a
         conditional target index.

        .. seealso::

            :ref:`postgresql_insert_on_conflict`

        """
        self._post_values_clause = OnConflictDoNothing(
            constraint, index_elements, index_where
        )
        return self

class OnConflictClause(ClauseElement):
    stringify_dialect = "postgresql"

    constraint_target: Optional[str]
    inferred_target_elements: _OnConflictIndexElementsT
    inferred_target_whereclause: _OnConflictIndexWhereT

    def __init__(
        self,
        constraint: _OnConflictConstraintT = None,
        index_elements: _OnConflictIndexElementsT = None,
        index_where: _OnConflictIndexWhereT = None,
    ):
        if constraint is not None:
            if not isinstance(constraint, str) and isinstance(
                constraint,
                (schema.Constraint, ext.ExcludeConstraint),
            ):
                constraint = getattr(constraint, "name") or constraint

        if constraint is not None:
            if index_elements is not None:
                raise ValueError(
                    "'constraint' and 'index_elements' are mutually exclusive"
                )

            if isinstance(constraint, str):
                self.constraint_target = constraint
                self.inferred_target_elements = None
                self.inferred_target_whereclause = None
            elif isinstance(constraint, schema.Index):
                index_elements = constraint.expressions
                index_where = constraint.dialect_options["postgresql"].get(
                    "where"
                )
            elif isinstance(constraint, ext.ExcludeConstraint):
                index_elements = constraint.columns
                index_where = constraint.where
            else:
                index_elements = constraint.columns
                index_where = constraint.dialect_options["postgresql"].get(
                    "where"
                )

        if index_elements is not None:
            self.constraint_target = None
            self.inferred_target_elements = index_elements
            self.inferred_target_whereclause = index_where
        elif constraint is None:
            self.constraint_target = (
                self.inferred_target_elements
            ) = self.inferred_target_whereclause = None

class OnConflictDoNothing(OnConflictClause):
    __visit_name__ = "on_conflict_do_nothing"

class OnConflictDoUpdate(OnConflictClause):
    __visit_name__ = "on_conflict_do_update"

    def __init__(
        self,
        constraint: _OnConflictConstraintT = None,
        index_elements: _OnConflictIndexElementsT = None,
        index_where: _OnConflictIndexWhereT = None,
        set_: _OnConflictSetT = None,
        where: _OnConflictWhereT = None,
    ):
        super().__init__(
            constraint=constraint,
            index_elements=index_elements,
            index_where=index_where,
        )

        if (
            self.inferred_target_elements is None
            and self.constraint_target is None
        ):
            raise ValueError(
                "Either constraint or index_elements, "
                "but not both, must be specified unless DO NOTHING"
            )

        if isinstance(set_, dict):
            if not set_:
                raise ValueError("set parameter dictionary must not be empty")
        elif isinstance(set_, ColumnCollection):
            set_ = dict(set_)
        else:
            raise ValueError(
                "set parameter must be a non-empty dictionary "
                "or a ColumnCollection such as the `.c.` collection "
                "of a Table object"
            )
        self.update_values_to_set = [
            (coercions.expect(roles.DMLColumnRole, key), value)
            for key, value in set_.items()
        ]
        self.update_whereclause = where


import urllib
from bs4 import BeautifulSoup
import re
import csv

url = "http://stats.footballpredictions.net/england/premier/1995-1996/results.html"
htmlfile = urllib.urlopen(url)
soup = BeautifulSoup(htmlfile)

#home team names
hometeam = soup.find_all("td", class_="hometeam")
ht = []
for element in hometeam:
	ht.append(element.a.get_text())

#away team names
awayteam = soup.find_all("td", class_="awayteam")
wt = []
for element in awayteam:
	wt.append(element.a.get_text())

#scores
scores = soup.find_all("td", class_="score")
sc = []
for element in scores:
	#sc.append(element.get_text())
	a = element.get_text()
	a = re.sub(r"\s+", "", a, flags=re.UNICODE)
	a = a.encode('ascii')
	sc.append(a.split('-'))

f = open("results.csv", "wt")

try:
	writer = csv.writer(f, delimiter=',')
	writer.writerow(("hometeam", "awayteam", "homegoal", "awaygoal"))
	for i in range(len(ht)):
		writer.writerow((ht[i], wt[i], sc[i][0], sc[i][1]))
finally:
	f.close()

f = open("results2.csv", "wt")

try:
	writer = csv.writer(f, delimiter=',')
	writer.writerow(("team", "opponent", "goal", "home"))
	for i in range(len(ht)):
		writer.writerow((ht[i], wt[i], sc[i][0], 1))
		writer.writerow((wt[i], ht[i], sc[i][1], 0))
finally:
	f.close()


# 784. Letter Case Permutation
# https://leetcode.com/problems/letter-case-permutation/
class Solution:
    def letterCasePermutation(self, s: str) -> List[str]:
        queue = []
        queue.append(s)

        for i in range(len(s)):
            if s[i].isalpha():
                for _ in range(len(queue)):
                    word = queue.pop(0)
                    left = word[:i]
                    right = word[i+1: len(word)]
                    queue.append(left + s[i].lower() + right)
                    queue.append(left + s[i].upper() + right)
        return queue


"""
A script that compares image histograms quantitively. The user must specify either a single image or a directory (jpg/png).
"""
# system tools
import os
import argparse
import sys

# image and data tools
import cv2
import numpy as np
import glob
import pandas as pd

# plotting tools
import matplotlib.pyplot as plt
import matplotlib.image as mpimg

# function that specifies the required arguments
def parse_args():
    # Initialise argparse
    ap = argparse.ArgumentParser()

    # command line parameters
    ap.add_argument("-f", "--file_input", required = True, help = "The filename or directory we want to work with")

    args = vars(ap.parse_args())
    return args

# function to clean a file path name
def clean_name(name):
    split_filename = name.split("/")[-1]
    file_only = split_filename.split(".")[0]

    return file_only

# function to create the histogram
def create_histogram(image):
     # creating histograms for image
    hist = cv2.calcHist([image], [0,1,2], None, [8,8,8], [0,256, 0,256, 0,256])
    hist = cv2.normalize(hist, hist, 0,255, cv2.NORM_MINMAX) #normalizing with MINMAX

    return hist

# function to calculate similarity scores
def calc_sim_scores(filename):
    # parse argument
    args = parse_args()
    input_name = args['file_input']

     # if the path provided is a file
    isFile = os.path.isfile(input_name)
    if isFile == True:
        input_name = input_name.split("/")[:-1]
        input_name = '/'.join(input_name) # remove the filename to get the directory

    # get all full path names
    file_list = glob.glob(os.path.join(input_name, '*g'))
    file_list = sorted(file_list)

    target_image = mpimg.imread(filename) # loading target image
    target_hist = create_histogram(target_image)

    # initiating lists for results
    sim_value = []
    img_name = []

    for img in file_list:
        if img != filename: # excluding the target image
            image = mpimg.imread(img)
            hist = create_histogram(image)

            score = round(cv2.compareHist(target_hist, hist, cv2.HISTCMP_CHISQR), 2) # comparing each histogram to the target image
            img_name.append(img) # appending results to the lists
            sim_value.append(score)

    # zipping the lists together, sorting it, and selecting the top 3 results.
    scores = sorted(zip(sim_value, img_name))
    scores = scores[0:3]

    return scores

# function to create the plot and dataframe of the results
def hist_result(file, scores):
    target_image = mpimg.imread(file) # loading target image
    file_name = clean_name(file)

    # initiating a list for the output dataframe
    output_data = []
    output_data.append(file_name)

    # initiating plot
    plt.subplot(1, 4, 1) #the figure has 1 row, 4 columns, and this is the first plot
    plt.imshow(target_image, extent=[-4, 4, -1, 1], aspect=4) # aligning the size of the images
    plt.axis('off')
    plt.title('Main image')

    plot_number = 1 # creating counter for the subplots

    for value, image in scores:
        value = round(value) # rounding the similarity score for a cleaner output
        image_name = clean_name(image)

        output_data.append(image_name) # saving the results of each image in the output list
        output_data.append(value)

        # plotting
        plot_number += 1
        image_sub = mpimg.imread(image)

        # creating the subplots
        plt.subplot(1, 4, plot_number)
        plt.imshow(image_sub, extent=[-4, 4, -1, 1], aspect=4)
        plt.axis('off')
        plt.title(f"Score: {value}")

    # Saving the plot
    plt.suptitle("Most similar images")
    plt.savefig(f'output/hist_{file_name}.jpg')

    # creating the dataframe and transposing it to get columns
    df = pd.DataFrame(output_data, index = ["main_image", "1_image", "1_image_score", "2_image", "2_image_score", "3_image", "3_image_score"])
    df = df.transpose()

    outpath = os.path.join('output', f'hist_{file_name}.csv')
    df.to_csv(outpath, index = False)

    return

# function to create a dataframe of the similarity scores for multiple input files
def hist_all_results(input_name):

    # get all full path names
    file_list = glob.glob(os.path.join(input_name, '*g'))
    file_list = sorted(file_list)

    # initiate list for dataframe
    out_list = []
    for file in file_list:
        scores = calc_sim_scores(file)
        file_name = clean_name(file)

        output_data = [] # initiating output data for each main image
        output_data.append(file_name)

        # cleaning the values and names, and appending them to the output data
        for value, image in scores:
            value = round(value)
            image_name = clean_name(image)

            output_data.append(image_name)
            output_data.append(value)

        out_list.append(output_data) # appending the output data to the out_list for the dataframe

    # creating the dataframe
    df = pd.DataFrame(out_list, columns = ["main_image", "1_image", "1_image_score", "2_image", "2_image_score", "3_image", "3_image_score"])

    outpath = os.path.join('output', 'hist_all_files.csv')
    df.to_csv(outpath)

    return

def main():

    # parse arguments
    args = parse_args()
    input_name = args['file_input']

    # if path provided is a file:
    isFile = os.path.isfile(input_name)
    if isFile == True:
        scores = calc_sim_scores(input_name)
        hist_result(input_name, scores)

        print('Input is a file. Script success.')

    # if path provided is a directory:
    isDirectory = os.path.isdir(input_name)
    if isDirectory == True:
        hist_all_results(input_name)

        print('Input is a directory. Script success.')

    return

if __name__ == '__main__':
    main()


from django.contrib.auth.models import User
from django.db import models
from django.db.models import Index

class EquipmentType(models.Model):
    """
        Тип оборудования
    """
    name = models.CharField(max_length=255, verbose_name='Type name')
    serial_number_mask = models.CharField(max_length=50, verbose_name='Mask of serial numbers')

    def __str__(self):
        return self.name

    class Meta:
        verbose_name = 'Type of equipment'
        verbose_name_plural = 'Type of equipment'
        ordering = ['name']
        indexes = (
            Index(fields=['name'], name='equ_typ__name__idx'),
        )
        constraints = [
            models.UniqueConstraint(
                fields=['name'],
                name='equ_typ__name__unq'
            ),
        ]

class Equipment(models.Model):
    """
        Оборудование
    """
    equipment_type = models.ForeignKey('EquipmentType', on_delete=models.PROTECT, related_name='equipments',
                                       verbose_name='Type of equipment')
    serial_number = models.CharField(max_length=50, verbose_name='Serial number of equipment')
    description = models.TextField(verbose_name='Description of equipment', null=True, blank=True)
    created_at = models.DateTimeField(auto_now_add=True, verbose_name='Created at')
    updated_at = models.DateTimeField(auto_now=True, verbose_name='Updated at')
    is_archived = models.BooleanField(default=False, verbose_name='Is archived')
    created_by = models.ForeignKey(User, on_delete=models.PROTECT, related_name='created_by',
                                   null=True, blank=True, verbose_name='Created by')
    updated_by = models.ForeignKey(User, on_delete=models.PROTECT, related_name='updated_by',
                                   null=True, blank=True, verbose_name='Updated by')

    class Meta:
        verbose_name = 'Equipment'
        verbose_name_plural = 'Equipment'
        ordering = ['id']
        indexes = (
            Index(
                fields=['is_archived', 'equipment_type', 'serial_number'],
                name='equ__type_serial_number__idx'
            ),
        )
        constraints = [
            models.UniqueConstraint(
                fields=['equipment_type', 'serial_number'],
                name='equ__type_serial_number__unq'
            ),
        ]


##diagonalize
import numpy as np
from qe.ReadDynMat import *
import cmath

def Get_Eval_Evec_dDdV ( Darray):

	Deq = []   ## equilibrium dyn matrices
	Dn = []		## non-equilibrium dyn matrices
	dD = []   ## what will be returned (dD/dV)
	eval = []	## eigenvalues (omega^2)
	evec = []	## eigenvectors that diagonalize eq matrices

	for x in xrange( len(Darray) ):  ## for each calculation there will be a subarray

		if x != 0:  ## i.e. not an equilibrium matrix

			Dn.append([])

		for y in xrange( len(Darray[x]) ): ## for each q point there will be a subarray

			if x == 0:  ## i.e. if we are looking at the equilibrium matrices

				#evec.append([])
				eval.append([])

				Deq.append ( Darray[0][y] )  ## the format needs to be that the EQUILIBRIUM dimension
							 				 ## is imported FIRST

				evali, eveci = np.linalg.eigh(Darray[0][y])

				eveci = np.asarray( eveci )

				for z in xrange( len(evali) ):
					eval[y].append( evali[z] )  ## populate eigenvalues

				# before i had evec[y].append but i think that gives wrong dimensions
				evec.append( eveci ) ## populate eigenvectors (3N x 3N matrix)

			else:  ## i.e. non-equilibrium matrices

				Dn[x-1].append(Darray[x][y])  ## this is a full dyn matrix

	# Dn array should have 2 columns of elements (for + and - volume calculations)
	#	, with each element being a 3Nx3N matrix of eigenvalues at that q-point
	# Deq array should have three dimensions

	for i in xrange(len(Deq)):  # q points

		dD.append([])

		for j in xrange( len(Deq[i]) ):  # 3N

			dD[i].append([])

			for k in xrange( len(Deq[i][j]) ):  # 3N

				dD[i][j].append( (Dn[0][i][j][k] - Dn[1][i][j][k]) )
				# Dn[0] is D+ , Dn[1] is D-

	return eval, evec, dD

## Calculate Gruneisen parameter

def GruCalc ( V, nat, amass, dir ):

	q, Darray = ReadDynMat ( nat, amass, dir)  # get dynamical matrices (eq, +, and -)

	eval_eq, evec_eq, dD = Get_Eval_Evec_dDdV ( Darray)  # get equilibrium eigenvalues
												   # and eigenvectors as well as the
												   # dD matrix

	gru = []
	gru.append([]) # for the q point column

	for i in xrange( len(dD) ):  ## should be equal to the number of q points
		gru[0].append(i+1) # append q point number (not the actual q point)

		# SORTING of eigenvalues: must correspond to eigenvectors whose inner product
		# with that of previous q point is 1
		#  e.g. < eig1(q=0) | eig1(q=1) >   = 1
		#       < eig1(q=0) | eig2(q=1) >   = 0

		index = []
		if i > 0:

			for x in xrange( len(evec_eq[i]) ):

				temp = []

				for y in xrange( len(evec_eq[i-1]) ):

					# Project, i.e. take the inner product of the two vectors
					proj = np.dot( evec_eq[i][:,x].transpose() , evec_eq[i-1][:,y] )
					temp.append( [abs(proj) , x, y] )

				index.append( max(temp)  ) # Take the max; should be close to 0

		########

		if i == 0:  # the first q point

			for j in xrange( len(dD[i]) ):  ## 3N

				gru.append([])

				evec_0 = np.asarray( evec_eq[i] )  # 3Nx3N matrix

				bra = evec_0[:,j]
				ket = evec_0[:,j].transpose()

				d_eval = np.dot( bra, (np.dot(dD[i] , ket) ) ).real

			# throw away very low values of d_omega^2 because these won't contribute to
			# thermal conductivity and they will result in large, misleading Gruneisen parameters

				if abs(d_eval) < 1e-8:  # AND abs(eval_eq[i][j]) < ~ 0.3 THz
					g = 0.0

				else:
					g = - ( d_eval/(2*eval_eq[i][j]) )* ( V[0]/(V[1]-V[2]))

				gru[j+1].append( float(g) )

			# Note: V elements MUST correspond to the correct calculations
			# V[0] = Veq ; V[1] = V+ ; V[2] = V-
			# I may want to change this later...

		else: # all subsequent q points: need to use the index variable for sorting
			for j in xrange( len(dD[i]) ): # 3N

				for x in xrange( len(index) ):
					if index[x][2] == j:

						evec_q = np.asarray( evec_eq[i])
						bra = evec_q[:,x]
						ket = evec_q[:,x].transpose()

						d_eval = np.dot( bra, (np.dot(dD[i] , ket) ) ).real

						if abs(d_eval) < 1e-8 and abs(eval_eq[i][j]) < 1e-7:
							g = 0.0

						else:
							g = - ( d_eval/(2*eval_eq[i][j]) )* ( V[0]/(V[1]-V[2]))

						gru[j+1].append( float(g) )

	return q, gru

''' ARCHIVE

def Get_Eval_Evec_dDdV ( Darray, V):

	# MAY NOT NEED TO CONVERT UNITS
	# convert Volume:  Bohr^3 to A^3
	#for i in range(len(V)):
	#	V[i] *=  (0.52917720859e-10)**3

	Deq = []   ## equilibrium dyn matrices
	Dn = []		## non-equilibrium dyn matrices
	dDdV = []   ## what will be returned (dD/dV)
	delDdelV = []  ## the average of dDdV, this one will be returned
	eval = []	## eigenvalues (omega^2)
	evec = []	## eigenvectors that diagonalize eq matrices

	for x in xrange( len(Darray) ):  ## for each calculation there will be a subarray

		if x != 0:  ## i.e. not an equilibrium matrix

			Dn.append([])

		for y in xrange( len(Darray[x]) ): ## for each q point there will be a subarray

			if x == 0:  ## i.e. if we are looking at the equilibrium matrices

				evec.append([])
				eval.append([])

				Deq.append ( Darray[0][y] )  ## the format needs to be that the EQUILIBRIUM dimension
							 				 ## is imported FIRST

				evali, eveci = np.linalg.eig(Darray[0][y])

				eveci = np.asarray( eveci )

				for z in xrange( len(evali) ):
					eval[y].append( evali[z] )  ## populate eigenvalues

				evec[y].append( eveci ) ## populate eigenvectors (3N x 3N matrix)

			else:  ## i.e. non-equilibrium matrices

				Dn[x-1].append(Darray[x][y])  ## this is a full dyn matrix

	# Dn array should have four dimensions,
	# Deq array should have three dimensions

	for x in xrange( len(Dn) ):	# dimensions including NON-equilibrium calculations

		dDdV.append([])

		for i in xrange( len(Deq) ):

			dDdV[x].append([])

			if x==0:
				delDdelV.append([])

			for j in xrange( len(Deq[i]) ):

				dDdV[x][i].append([])

				if x==0:
					delDdelV[i].append([])

				for k in xrange( len(Deq[i][j]) ):  ## populate dD/dV matrix values

					dDdV[x][i][j].append( (Dn[x][i][j][k] - Deq[i][j][k])/( V[x+1]-V[0] ) )

					if x==0:
						delDdelV[i][j].append(0.0)

	for x in xrange( len(dDdV) ):

		for i in xrange( len(delDdelV) ):

			for j in xrange( len(delDdelV[i]) ):

				for k in xrange( len(delDdelV[i][j]) ):

					delDdelV[i][j][k] += dDdV[x][i][j][k]

					if x == len(dDdV)-1:

						delDdelV[i][j][k] /= (x+1)

	return eval, evec, delDdelV

def GruCalc ( V, nat, amass, dir ):

	Darray = ReadDynMat ( nat, amass, dir)

	eval, evec, dD = Get_Eval_Evec_dDdV ( Darray)

	gru = []
	gru.append([]) # for q point column

	for i in xrange( len(dD) ):  ## should be equal to the number of q points
		gru[0].append(i+1)

		for j in xrange( len(dD[i]) ):  ## 3N

			if i==0:  # happens 3N times
				gru.append([])

			eigvec = np.asarray( evec[i] )

			bra = eigvec[:,j]
			ket = eigvec[:,j].transpose()

			d_eval = np.dot( bra, (np.dot(dD[i] , ket) ) ).real

			# Note: V elements MUST correspond to the correct calculations
			# V[0] = Veq ; V[1] = V+ ; V[2] = V-

			# throw away very low values of d_omega^2 because these won't contribute to
			# thermal conductivity and they will result in large, misleading Gruneisen parameters
			if abs(d_eval) < 1e-8:
				g = 0.0

			else:
				g = - ( d_eval/(2*eval[i][j]) )* ( V[0]/(V[1]-V[2]))

			gru[j+1].append( float(g) )

	return gru
'''


import ipaddress
import re
import glob

def Return_IP_adr(str):
    if re.match("^ ip address ([0-9.]+) ([0-9.]+)$", str):
        r = re.match("^ ip address ([0-9.]+) ([0-9.]+)$", str)
        return ipaddress.IPv4Network((r.group(1), r.group(2)),strict = False)
    else:
        return "None"

list_files = glob.glob("C:\\Users\\nv.solomennikova\\Documents\\pythonProject\\p4ne\\Lab1.5\\config_files\\*.txt")
list_result = []
for fl in list_files:
    with open (fl) as f:
        for s in f:
            res = Return_IP_adr(s)
            if res != "None":
                list_result.append(res)

print((list(set(list_result))))

'''
r = re.match("^ ip address ([0-9.]+) ([0-9.]+)$", " ip address 192.168.1.1 255.255.255.0")
print(r.group(2))
res = ipaddress.IPv4Network((r.group(1), r.group(2)),strict = False)

'''


import requests
from bs4 import BeautifulSoup
import pandas as pd

def extract(page):
    url = f'https://in.indeed.com/jobs?q=python+developer&l=India&start={page}'
    res = requests.get(url)
    # return res.status_code
    soup = BeautifulSoup(res.content, 'html.parser')
    return soup

job_list = []

def transfrom(soup):
    divs = soup.find_all('div', class_='jobsearch-SerpJobCard')

    for i in divs:
        title = i.find('a').text.strip()
        print(title)
        company = i.find('span', class_='company').text.strip()
        print(company)
        summary = i.find(
            'div', class_='summary').text.strip().replace('\n', '')
        print(summary)

        job = {
            'title': title,
            "company": company,
            "summary": summary
        }
        job_list.append(job)

for i in range(0, 50, 10):
    print(f'Getting Page,{i}')

    c = extract(0)
    transfrom(c)

# print(len(job_list))

df = pd.DataFrame(job_list)
print(df.head())
df.to_csv('indeed.csv')


import time
import datetime
from grooveshark import Grooveshark

################################################################################
TITLE          = 'Grooveshark'
ART            = 'art-default.jpg'
ICON           = 'icon-default.png'
SEARCH_ICON    = 'icon-search.png'
PREFS_ICON     = 'icon-prefs.png'
PREFIX         = '/music/grooveshark'
shark          = Grooveshark()

def toInt(s):
    try:
        return int(s)
    except ValueError:
        return int(float(s))

def sortInt(s):
    if s == None or s == '0' or s == '1901':
        return 10000
    return int(s)

################################################################################
def Start():
    DirectoryObject.thumb  = R(ICON)
    ObjectContainer.art    = R(ART)
    ObjectContainer.title1 = TITLE
    ArtistObject.thumb     = R(ICON)
    AlbumObject.thumb      = R(ICON)
    TrackObject.thumb      = R(ICON)

################################################################################
@handler(PREFIX, TITLE, art=ART, thumb=ICON)
def Main():
    oc = ObjectContainer(title2=TITLE)

    if Prefs['username'] and Prefs['password']:
        shark.authenticateUser(Prefs['username'], Prefs['password'])
        if shark.isAuthenticated():
            oc.add(DirectoryObject(key=Callback(Collection), title=L('Collection')))
            oc.add(DirectoryObject(key=Callback(Favorites), title=L('Favorites')))
            oc.add(DirectoryObject(key=Callback(Playlists), title=L('Playlists')))

    oc.add(DirectoryObject(key=Callback(Genres), title=L('Genres')))
    oc.add(DirectoryObject(key=Callback(Broadcasts), title=L('Broadcasts')))
    oc.add(DirectoryObject(key=Callback(Popular), title=L('Popular')))
    oc.add(InputDirectoryObject(key=Callback(Search), title=L('Search'), prompt=L('Search for'), thumb=R(SEARCH_ICON)))
    oc.add(PrefsObject(title=L('Preferences'), thumb=R(PREFS_ICON)))
    return oc

################################################################################
@route(PREFIX + '/collection', page=int)
def Collection(page=0):
    oc = ObjectContainer(title2=L('Collection'))

    library = shark.userGetSongsInLibrary(page)
    for song in sorted(library['Songs'], key = lambda x: (x.get('ArtistName', None), x.get('AlbumName', None), sortInt(x.get('TrackNum')))):
        oc.add(CreateTrackObject(song=song))

    if library['hasMore'] == True:
        oc.add(NextPageObject(key=Callback(Collection, page=page+1)))

    return oc

################################################################################
@route(PREFIX + '/favorites')
def Favorites():
    oc = ObjectContainer(title2=L('Favorites'))

    favorites = shark.getFavorites()
    for song in sorted(favorites, key = lambda x: (x.get('ArtistName', None), x.get('AlbumName', None), sortInt(x.get('TrackNum')))):
        oc.add(CreateTrackObject(song=song))

    return oc

################################################################################
@route(PREFIX + '/playlists')
def Playlists():
    oc = ObjectContainer(title2=L('Playlists'))

    playlists = shark.userGetPlaylists()
    for playlist in playlists['Playlists']:
        do = DirectoryObject(key = Callback(PlaylistsSubMenu, title=playlist['Name'], id=playlist['PlaylistID']), title=playlist['Name'])
        if 'Picture' in playlist and playlist['Picture'] != None:
            do.thumb = shark.playlist_base_url + '200_' + playlist['Picture']
        else:
            do.thumb = shark.no_album_url

        oc.add(do)

    return oc

################################################################################
@route(PREFIX + '/genres')
def Genres():
    oc = ObjectContainer(title2=L('Genres'))

    tags = shark.getTopLevelTags()
    for tag in tags:
        oc.add(DirectoryObject(key = Callback(GenreSubMenu, title=tag['Tag'], id=tag['TagID']), title=tag['Tag']))

    return oc

################################################################################
@route(PREFIX + '/broadcasts')
def Broadcasts():
    oc = ObjectContainer(title2=L('Broadcasts'))

    broadcasts = shark.getTopBroadcastsCombined()
    for key, value in sorted(broadcasts.iteritems(), key = lambda x: ('subscribers_count' in x[1], x[1].get('subscribers_count')), reverse = True):
        if 'n' in value and 's' in value:
            if 'active' in value['s']:
                if 'b' in value['s']['active']:
                    if 'tk' in value['s']['active']['b'] and value['s']['active']['b']['tk']:
                        song = {'SongID': value['s']['active']['b']['tk'],
                                'ArtistName': value['n'],
                                'Name': value['s']['active']['b']['sN'] + ' by ' + value['s']['active']['b']['arN'],
                                'BroadcastId': key.split(':')[1],
                                'CoverArtFilename': shark.no_user_url,
                                'EstimateDuration': None}

                        if 'i' in value and value['i'] != None:
                            song['CoverArtFilename'] = shark.broadcast_base_url + value['i']
                        elif 'users' in value and len(value['users']) > 0 and 'Picture' in value['users'][0] and value['users'][0]['Picture'] != None:
                            song['CoverArtFilename'] = shark.users_base_url + value['users'][0]['Picture']

                        oc.add(CreateTrackObject(song=song))

    return oc

################################################################################
@route(PREFIX + '/popular')
def Popular():
    oc = ObjectContainer(title2=L('Popular'))
    oc.add(DirectoryObject(key = Callback(PopularSubMenu, title=L('Popular Today'), type='daily'), title=L('Popular Today')))
    oc.add(DirectoryObject(key = Callback(PopularSubMenu, title=L('Popular This Week'), type='weekly'), title=L('Popular This Week')))
    oc.add(DirectoryObject(key = Callback(PopularSubMenu, title=L('Popular This Month'), type='monthly'), title=L('Popular This Month')))
    return oc

################################################################################
@route(PREFIX + '/playlistssubmenu')
def PlaylistsSubMenu(title, id):
    oc = ObjectContainer(title2=title)

    songs = shark.playlistGetSongs(id)
    for song in songs['Songs']:
            oc.add(CreateTrackObject(song=song))

    return oc

################################################################################
@route(PREFIX + '/genresubmenu')
def GenreSubMenu(title, id):
    oc = ObjectContainer(title2=title)
    oc.add(DirectoryObject(key = Callback(GenrePlayMenu, title=title, id=id), title=L('Play') + ' ' + title))
    oc.add(DirectoryObject(key = Callback(GenrePlayMenu, title=L('Related Genres') + ': ' + title, id=id, type='related'), title=L('Related Genres')))
    return oc

################################################################################
@route(PREFIX + '/genreplaymenu')
def GenrePlayMenu(title, id, type=None):
    oc = ObjectContainer(title2=title)

    info = shark.getPageInfoByIDType(id)
    if type == 'related':
        for song in info['Data']['RelatedTags']:
            oc.add(DirectoryObject(key = Callback(GenrePlayMenu, title=song['TagName'], id=song['TagID']), title=L('Play') + ' ' + song['TagName']))

    else:
        for song in info['Data']['Songs']:
            oc.add(CreateTrackObject(song=song))

    return oc

################################################################################
@route(PREFIX + '/popularsubmenu')
def PopularSubMenu(title, type):
    oc = ObjectContainer(title2=title)

    songs = shark.popularGetSongs(type)
    for song in songs['Songs']:
        oc.add(CreateTrackObject(song=song))

    return oc

################################################################################
@route(PREFIX + '/search')
def Search(query):
    oc = ObjectContainer(title2=L('Search Results'))

    results = shark.getResultsFromSearch(query)
    for key, values in results['result'].iteritems():
        if key == 'Artists':
            for artist in values:
                artistObj = ArtistObject(
                    key=Callback(ShowArtistOptions, name=artist['Name'], id=artist['ArtistID']),
                    rating_key=artist['ArtistID'],
                    title=artist['Name']
                )

                if 'CoverArtFilename' in artist and artist['CoverArtFilename'] != None and ''.join(artist['CoverArtFilename'].split()) != '':
                    artistObj.thumb=shark.artist_base_url + artist['CoverArtFilename']
                else:
                    artistObj.thumb=shark.no_artist_url

                oc.add(artistObj)

        elif key == 'Songs':
            for song in values:
                oc.add(CreateTrackObject(song=song))

        elif key == 'Albums':
            for album in values:
                albumObj = AlbumObject(
                    key=Callback(ShowAlbumOptions, name=album['AlbumName'], id=album['AlbumID']),
                    rating_key=album['AlbumID'],
                    artist=album['ArtistName'],
                    title=album['AlbumName']
                )

                if 'CoverArtFilename' in album and album['CoverArtFilename'] != None and ''.join(album['CoverArtFilename'].split()) != '':
                    albumObj.thumb=shark.album_base_url + album['CoverArtFilename']
                else:
                    albumObj.thumb=shark.no_album_url

                oc.add(albumObj)

    return oc

################################################################################
@route(PREFIX + '/showartistoptions')
def ShowArtistOptions(name, id):
    oc = ObjectContainer(title2=name)

    albums = shark.artistGetAllAlbums(id)
    for album in sorted(albums['albums'], key = lambda x: sortInt(x.get('Year'))):
        albumObj = AlbumObject(
            key=Callback(ShowAlbumOptions, name=album['Name'], id=album['AlbumID']),
            rating_key=album['AlbumID'],
            artist=name,
            title=album['Name']
        )

        if 'CoverArtFilename' in album and album['CoverArtFilename'] != None and ''.join(album['CoverArtFilename'].split()) != '':
            albumObj.thumb=shark.album_base_url + album['CoverArtFilename']
        else:
            albumObj.thumb=shark.no_album_url

        oc.add(albumObj)

    return oc

################################################################################
@route(PREFIX + '/showalbumoptions')
def ShowAlbumOptions(name, id):
    oc = ObjectContainer(title2=name)

    songs = shark.albumGetAllSongs(id)
    for song in sorted(songs, key = lambda x: sortInt(x.get('TrackNum'))):
        oc.add(CreateTrackObject(song=song))

    return oc

################################################################################
@route(PREFIX + '/createtrackobject', song=dict)
def CreateTrackObject(song, include_container=False):
    media_obj = MediaObject(
        audio_codec = AudioCodec.MP3,
        container = 'mp3'
    )
    track_obj = TrackObject(
        key = Callback(CreateTrackObject, song=song, include_container=True),
        rating_key = song['SongID']
    )

    if 'Name' in song:
        track_obj.title = song['Name']
    elif 'SongName' in song:
        track_obj.title = song['SongName']
    else:
        track_obj.title = L('No title provided')

    if 'BroadcastId' in song:
        media_obj.add(PartObject(key = Callback(GetBroadcastURL, id=song['BroadcastId'], ext='mp3')))
    else:
        media_obj.add(PartObject(key = Callback(GetStreamURL, id=song['SongID'], ext='mp3')))

    if 'ArtistName' in song and song['ArtistName'] != None:
        track_obj.artist = song['ArtistName']

    if 'AlbumName' in song and song['AlbumName'] != None:
        track_obj.album = song['AlbumName']

    if 'TrackNum' in song and song['TrackNum'] != None:
        track_obj.index = int(song['TrackNum'])

    if 'EstimateDuration' in song and song['EstimateDuration'] != None:
        track_obj.duration = toInt(song['EstimateDuration']) * 1000

    if 'CoverArtFilename' in song and song['CoverArtFilename'] != None and ''.join(song['CoverArtFilename'].split()) != '':
        if song['CoverArtFilename'].startswith('http'):
            track_obj.thumb = song['CoverArtFilename']
        else:
            track_obj.thumb = shark.album_base_url + song['CoverArtFilename']
    else:
        track_obj.thumb = shark.no_album_url

    track_obj.add(media_obj)

    if include_container:
        return ObjectContainer(objects=[track_obj])
    else:
        return track_obj

################################################################################
@route(PREFIX + '/getstreamurl')
def GetStreamURL(id):
    url, server, key = shark.getStreamKeyFromSongIDEx(id)
    if url:
        Thread.Create(MarkSongs, id=id, server=server, key=key)
        return Redirect(url)
    else:
        return Redirect('')

################################################################################
@route(PREFIX + '/getbroadcasturl')
def GetBroadcastURL(id):
    url = shark.getMobileBroadcastURL(id, Prefs['broadcast_quality'])

    if url == None:
        url = shark.getMobileBroadcastURL(id, Prefs['broadcast_quality'])

    return Redirect(url)

########################## Thread Function #####################################
def MarkSongs(id, server, key):
    shark.markSongDownloadedEx(id, server, key)
    time.sleep(2)

    shark.markSongQueueSongPlayed(id, server, key)
    time.sleep(30)

    shark.markStreamKeyOver30Seconds(id, server, key)
    time.sleep(30)

    shark.markSongComplete(id, server, key)
    return


import os
import queue
import json
from jsonschema import validate
import collections

class TestCase():
    """Attributes of a single test.

    Args:
        test_data (dict): data for a single test, parsed from JSON
                          test declaration.
    """
    __test__ = False  #: Ignored by Pytest

    def __init__(self, test_data):
        self.tag = test_data["tag"]

        #: list of couples of decl: List of IO declarations.
        self.io_decl = self._make_io_decl(test_data)

        #: int: ticks to pass before next input application.
        self.ticks_before_next_input = test_data["ticks_before_next_input"]

        #: int: ticks to pass before checking outputs
        self.ticks_before_checking = test_data["ticks_before_checking"]

    def _make_io_decl(self, test_data):
        """Make io_decl dictionaries from data.

        Args:
            test_data (dict): data for a single test parsed from a JSON tests
                              declaration file.

        Returns:
            list: a list of test declarations.
        """

        def compute_decl(decl):
            res = {}

            for signame, value in decl.items():
                if isinstance(value, str):
                    value = int(value, 2)
                res[signame] = value

            return res

        res = []

        for r in test_data["rounds"]:
            res += [(compute_decl(r["inputs"]), compute_decl(r["outputs"]))]

        return res

    def get_io_queues(self):
        """Generate input and expected io_decl queues.

        Args:
            testcase (TestDeclaration()): current test case.

        Returns:
            (queue.Queue(), queue.Queue()): input and expected declarations
                                            queues.
        """

        def _push_expected_values(q, is_first, expected_values):
            if is_first:
                for _ in range(self.ticks_before_checking):
                    q.put(None)
            else:
                for _ in range(self.ticks_before_next_input - 1):
                    q.put(None)

            q.put(expected_values)

            return q

        def _push_input_values(q, in_values):
            q.put(in_values)
            for _ in range(self.ticks_before_next_input - 1):
                q.put(None)

        in_q = queue.Queue()
        exp_q = queue.Queue()

        is_first = True
        for in_values, expected_values in self.io_decl:
            _push_input_values(in_q, in_values)
            _push_expected_values(exp_q, is_first, expected_values)
            is_first = False

        return in_q, exp_q

class TestsDeclaration():
    """Contains test declarations data.

    Args:
        filename (str): name of the test declaration JSON file.
    """

    __test__ = False  #: Ignored by Pytest

    def __init__(self, filename):
        #: dict: data parsed from JSON tests declaration.
        self._data = self._parse_testfile(filename)

        #: str: name of the testsuite.
        self.name = self._data["name"]

        #: list of TestAttributes(): list of each test's attributes.
        self.testcases = self._make_testcases()

    def _parse_testfile(self, filename):
        """Parses a JSON tests declaration from filename.

        Args:
            filename (str): name of the JSON file to parse.

        Returns:
            dict: data parsed from JSON file.
        """

        res = None

        with open(filename, 'r') as f:
            res = json.loads(
                f.read(), object_pairs_hook=collections.OrderedDict)

        self._validate_testfile(res)

        return res

    def _validate_testfile(self, data):
        """Validate raw data against JSON Scheme.

        Args:
            data (dict): data parsed from JSON file.
        """
        schema = None
        dir = os.path.dirname(os.path.realpath(__file__))
        with open(dir + "/testdecl_scheme.json", "r") as f:
            schema = json.loads(f.read())

        validate(data, schema)

    def _make_testcases(self):
        """Make a list of TestAttributes() from tests declaration.

        Returns:
            list: a list of TestAttributes().
        """
        res = []

        for t in self._data["tests"]:
            res += [TestCase(t)]

        return res


import argparse
import re
import os
import pickle

'''--------------------Parsing argumnts--------------------'''
parser = argparse.ArgumentParser(description="Этот код создаёт модель, обученную на текстах песен")
parser.add_argument(
    '--input_dir',
    type=str,
    help="Путь к папке с песнями"
)
parser.add_argument(
    '--model',
    type=str,
    default="model.pkl",
    help="Путь к файлу, в который сохраняется модель(с расширением pkl)"
)
args = parser.parse_args()
if args.input_dir==None:
    args.input_dir = input("Введите путь к папке с текстами: ")

'''--------------------Generating model--------------------'''
model = {}
data = os.listdir(path=args.input_dir)
for file in data:
    with open(args.input_dir+"/"+file,"r",encoding="utf-8") as file:
        text = ""
        for i in file:
            text += i
        text = re.sub(r'\W+',' ',text)
        text = text.lower().strip().split()
        for i in range(len(text)-1):
            if i>0:
                model[text[i-1]+" "+text[i]] = model.get(text[i-1]+" "+text[i],[])
                model[text[i-1]+" "+text[i]].append(text[i+1])
            model[text[i]] = model.get(text[i],[])
            model[text[i]].append(text[i+1])
'''--------------------Saving model--------------------'''
with open(args.model,"wb") as f:
    pickle.dump(model,f)


import os
import xml.etree.ElementTree as ET
import json

def append_barcode(results, barcode):
    result = {}
    result["attrib"] = barcode.attrib
    Values = get_elements(barcode, "Value")
    for Value in Values:
        result["text"] = Value.text
        result["value_attrib"] = Value.attrib
    results.append(result)

def get_elements(root, name):
    children = []
    for child in root:
        if child.tag == name:
            children.append(child)
    return children

for filename in os.listdir("./Markup"):
    if filename.endswith(".xml") == False:
        continue
    tree = ET.parse(os.path.join("./Markup",filename))
    root = tree.getroot()
    pages = root[0]
    results = []
    for page in pages:
        BarcodesRoot = get_elements(page, "Barcodes")[0]
        Barcodes = get_elements(BarcodesRoot, "Barcode")
        for barcode in Barcodes:
            append_barcode(results, barcode)
    name, ext = os.path.splitext(filename)
    txt_path = os.path.join("./Image",name+".txt")
    f = open (txt_path,"w")
    f.write(json.dumps(results))
    f.close()


from enum import IntEnum, auto
from typing import Tuple, Sequence

from unsserv.common.utils import parse_node
from unsserv.common.structs import Node
from unsserv.common.rpc.structs import Message
from unsserv.common.rpc.protocol import AProtocol, ITranscoder, Command, Data, Handler
from unsserv.extreme.searching.structs import WalkResult, Walk

FIELD_COMMAND = "kwalker-command"
FIELD_WALK_ID = "kwalker-walk-id"
FIELD_ORIGIN_NODE = "kwalker-origin-node"
FIELD_TTL = "kwalker-ttl"
FIELD_DATA_ID = "kwalker-data-id"
FIELD_WALK_RESULT = "kwalker-walk-result"

class KWalkerCommand(IntEnum):
    WALK = auto()
    WALK_RESULT = auto()

class KWalkerTranscoder(ITranscoder):
    def encode(self, command: Command, *data: Data) -> Message:
        if command == KWalkerCommand.WALK:
            walk: Walk = data[0]
            message_data = {
                FIELD_COMMAND: KWalkerCommand.WALK,
                FIELD_WALK_ID: walk.id,
                FIELD_DATA_ID: walk.data_id,
                FIELD_ORIGIN_NODE: walk.origin_node,
                FIELD_TTL: walk.ttl,
            }
            return Message(self.my_node, self.service_id, message_data)
        elif command == KWalkerCommand.WALK_RESULT:
            walk_result: WalkResult = data[0]
            message_data = {
                FIELD_COMMAND: KWalkerCommand.WALK_RESULT,
                FIELD_WALK_ID: walk_result.walk_id,
                FIELD_WALK_RESULT: walk_result.result,
            }
            return Message(self.my_node, self.service_id, message_data)
        raise ValueError("Invalid Command")

    def decode(self, message: Message) -> Tuple[Command, Sequence[Data]]:
        command = message.data[FIELD_COMMAND]
        if command == KWalkerCommand.WALK:
            walk = Walk(
                id=message.data[FIELD_WALK_ID],
                data_id=message.data[FIELD_DATA_ID],
                origin_node=parse_node(message.data[FIELD_ORIGIN_NODE]),
                ttl=message.data[FIELD_TTL],
            )
            return KWalkerCommand.WALK, [walk]
        elif command == KWalkerCommand.WALK_RESULT:
            walk_result = WalkResult(
                walk_id=message.data[FIELD_WALK_ID],
                result=message.data[FIELD_WALK_RESULT],
            )
            return KWalkerCommand.WALK_RESULT, [walk_result]
        raise ValueError("Invalid Command")

class KWalkerProtocol(AProtocol):
    def _get_new_transcoder(self):
        return KWalkerTranscoder(self.my_node, self.service_id)

    async def walk(self, destination: Node, walk: Walk):
        message = self._transcoder.encode(KWalkerCommand.WALK, walk)
        return await self._rpc.call_send_message(destination, message)

    async def walk_result(self, destination: Node, walk_result: WalkResult):
        message = self._transcoder.encode(KWalkerCommand.WALK_RESULT, walk_result)
        return await self._rpc.call_send_message(destination, message)

    def set_handler_walk(self, handler: Handler):
        self._handlers[KWalkerCommand.WALK] = handler

    def set_handler_walk_result(self, handler: Handler):
        self._handlers[KWalkerCommand.WALK_RESULT] = handler


import sys
import os
sys.path.insert(0,  os.getcwd() + '/../keggimporter')

import logging
import logging.handlers
from Config import *
from Importer import *

config = Config()
config.loadConfiguration()
conf = config.getConfigurations()

logFile = conf.get( 'log', 'info' )

log = logging.getLogger('')
log.setLevel(logging.DEBUG)
format = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")

ch = logging.StreamHandler(sys.stdout)
ch.setFormatter(format)
log.addHandler(ch)

fh = logging.handlers.RotatingFileHandler( logFile , maxBytes=0, backupCount=0)
fh.setFormatter(format)
log.addHandler(fh)

log.info('keggimporter: START')

imp = Importer()

imp.startImporter()

log.info('writeTaxonomies: START')
imp.writeTaxonomies()
log.info('writeTaxonomies: DONE')

log.info('writePathways: START')
imp.writePathways()
log.info('writePathways: DONE')

log.info('writeEcs: START')
imp.writeEcs()
log.info('writeEcs: DONE')

log.info('writeOrganisms: START')
imp.writeOrganisms()
log.info('writeOrganisms: DONE')

log.info('writeProteins: START')
imp.writeProteins()
log.info('writeProteins: DONE')

log.info('writeProteinRelations: START')
imp.writeProteinRelations()
log.info('writeProteinRelations: DONE')

log.info('writeOrganismTaxonomies: START')
imp.writeOrganismTaxonomies()
log.info('writeOrganismTaxonomies: DONE')

log.info('writeProteinAccessions: START')
imp.writeProteinAccessions()
log.info('writeProteinAccessions: DONE')

log.info('writeEcMaps: START')
imp.writeEcMaps()
log.info('writeEcMaps: DONE')

log.info('keggimporter: DONE')


import pandas as pd
import functest as ft
# 함수화 규칙
#  입력 데이터는 괄호안에 작성
#  데이터의 출력은 return
#  출력 관련 함수 사용금지

def getSortData(data,groupList):
    data1 = data.set_index(groupList).sort_index()
    return data1.reset_index(inplace=False)

data=pd.read_csv("ymd.csv",sep="\t|,",names=['year','month','day','data'])
groupList=['year','month','day']

data1 = getSortData(data,groupList)

# month + day 값이 필요함
def setMonthDay(data, maxRange, col1, col2, newcol):
    for i in range(maxRange):
        month=str(data.loc[i,col1])
        day=str(data.loc[i,col2])
        data.loc[i,newcol]=month+day
    return data

maxYear=4
maxDay=12*3
data2 = setMonthDay(data1,maxYear*maxDay,'month','day','monthday')

ft.showPlot(data2,maxYear,maxDay,'monthday','data')

print(data2)


import math

#Obtenção de Dados

matriz = [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]]

sub1 = sub2 = sub3 = sub4 = sub5 = sub6 = sub7 = sub8 = sub9 = sub10 = sub11 = sub12 = 0
linha1 = linha2 = linha3 = linha4 = linha5 = linha6 = 0
mid1 = mid2 = mid3 = mid4 = mid5 = mid6 = 0
histerese = 0

for l in range (0, 5):
    for c in range (0, 4):
        matriz[l][c]= float(input(f'Digite um valor para [{l}, {c}]: '))
print('-=' * 30)

for l in range(0, 5):
    for c in range(0, 4):
        print(f'[{matriz[l][c]:^5}]', end='')
    print()
print('-=' * 30)

#Incerteza da Histerese

#Subtração de colunas
sub1 = matriz[0][0] - matriz[0][1]
sub2 = matriz[0][2] - matriz[0][3]
sub3 = matriz[1][0] - matriz[1][1]
sub4 = matriz[1][2] - matriz[1][3]
sub5 = matriz[2][0] - matriz[2][1]
sub6 = matriz[2][2] - matriz[2][3]
sub7 = matriz[3][0] - matriz[3][1]
sub8 = matriz[3][2] - matriz[3][3]
sub9 = matriz[4][0] - matriz[4][1]
sub10 = matriz[4][2] - matriz[4][3]

#Comparar a maior histerese

maior = sub1

if sub2>sub1 and sub2>sub3 and sub2>sub4 and sub2>sub5 and sub2>sub6 and sub2>sub7 and sub2>sub8 and sub2>sub9 and sub2>sub10:
    maior = sub2

elif sub3>sub1 and sub3>sub2 and sub3>sub4 and sub3>sub5 and sub3>sub6 and sub3>sub7 and sub3>sub8 and sub3>sub9 and sub3>sub10:
    maior = sub3

elif sub4>sub1 and sub4>sub2 and sub4>sub3 and sub4>sub5 and sub4>sub6 and sub4>sub7 and sub4>sub8 and sub4>sub9 and sub4>sub10:
    maior = sub4

elif sub5>sub1 and sub5>sub2 and sub5>sub3 and sub5>sub4 and sub5>sub6 and sub5>sub7 and sub5>sub8 and sub5>sub9 and sub5>sub10:
    maior = sub5

elif sub6>sub1 and sub6>sub2 and sub6>sub3 and sub6>sub4 and sub6>sub5 and sub6>sub7 and sub6>sub8 and sub6>sub9 and sub6>sub10:
    maior = sub6

elif sub7>sub1 and sub7>sub2 and sub7>sub3 and sub7>sub4 and sub7>sub5 and sub7>sub6 and sub7>sub8 and sub7>sub9 and sub7>sub10:
    maior = sub7

elif sub8>sub1 and sub8>sub2 and sub8>sub3 and sub8>sub4 and sub8>sub5 and sub8>sub6 and sub8>sub7 and sub8>sub9 and sub8>sub10:
    maior = sub8

elif sub9>sub1 and sub9>sub2 and sub9>sub3 and sub9>sub4 and sub9>sub5 and sub9>sub6 and sub9>sub7 and sub9>sub8 and sub9>sub10:
    maior = sub9

elif sub10>sub1 and sub10>sub2 and sub10>sub3 and sub10>sub4 and sub10>sub5 and sub10>sub6 and sub10>sub7 and sub10>sub8 and sub10>sub9:
    maior = sub10

print(f'A maior histerese é: {maior}')

if maior<0:
    maior * (-1)

uHisterese = maior/math.sqrt(12)

#Incerteza do tipo A

#Cálculo da média

for c in range (0,4):
    linha1 += matriz[0][c]

for c in range (0,4):
    linha2 += matriz[1][c]

for c in range (0,4):
    linha3 += matriz[2][c]

for c in range (0,4):
    linha4 += matriz[3][c]

for c in range (0,4):
    linha5 += matriz[4][c]

#Média

mid1 = linha1/4
mid2 = linha2/4
mid3 = linha3/4
mid4 = linha4/4
mid5 = linha5/4

#Subtração das médias

#Média Linha 1
a1 = (mid1-matriz[0][0])
b1 = (mid1-matriz[0][1])
c1 = (mid1-matriz[0][2])
d1 = (mid1-matriz[0][3])

#Média Linha 2
a2 = (mid2-matriz[1][0])
b2 = (mid2-matriz[1][1])
c2 = (mid2-matriz[1][2])
d2 = (mid2-matriz[1][3])

#Média Linha 3
a3 = (mid3-matriz[2][0])
b3 = (mid3-matriz[2][1])
c3 = (mid3-matriz[2][2])
d3 = (mid3-matriz[2][3])

#Média Linha 4
a4 = (mid4-matriz[3][0])
b4 = (mid4-matriz[3][1])
c4 = (mid4-matriz[3][2])
d4 = (mid4-matriz[3][3])

#Média Linah 5
a5 = (mid5-matriz[4][0])
b5 = (mid5-matriz[4][1])
c5 = (mid5-matriz[4][2])
d5 = (mid5-matriz[4][3])

#Elevação ao quadrado dos valores

#Elevação Linha 1
ElevA1 = a1 ** 2
ElevB1 = b1 ** 2
ElevC1 = c1 ** 2
ElevD1 = d1 ** 2

#Elevação Linha 2
ElevA2 = a2 ** 2
ElevB2 = b2 ** 2
ElevC2 = c2 ** 2
ElevD2 = d2 ** 2

#Elevação Linha 3
ElevA3 = a3 ** 2
ElevB3 = b3 ** 2
ElevC3 = c3 ** 2
ElevD3 = d3 ** 2

#Elevação Linha 4
ElevA4 = a4 ** 2
ElevB4 = b4 ** 2
ElevC4 = c4 ** 2
ElevD4 = d4 ** 2

#Elevação Linha 5
ElevA5 = a5 ** 2
ElevB5 = b5 ** 2
ElevC5 = c5 ** 2
ElevD5 = d5 ** 2

#Soma dos valores de incerteza tipo A

Soma1 = ElevA1+ElevB1+ElevC1+ElevD1
Soma2 = ElevA2+ElevB2+ElevC2+ElevD2
Soma3 = ElevA3+ElevB3+ElevC3+ElevD3
Soma4 = ElevA4+ElevB4+ElevC4+ElevD4
Soma5 = ElevA5+ElevB5+ElevC5+ElevD5

#Divisão por raíz de 12

raiz1 = Soma1/math.sqrt(12)
raiz2 = Soma2/math.sqrt(12)
raiz3 = Soma3/math.sqrt(12)
raiz4 = Soma4/math.sqrt(12)
raiz5 = Soma5/math.sqrt(12)

#Maior incerteza do tipo A entre as linhas

uA = raiz1

if raiz2>raiz1 and raiz2>raiz3 and raiz2>raiz4 and raiz2>raiz5:
    uA = raiz2

elif raiz3>raiz1 and raiz3>raiz2 and raiz3>raiz4 and raiz3>raiz5:
    uA = raiz3

elif raiz4>raiz1 and raiz4>raiz2 and raiz4>raiz3 and raiz4 > raiz5:
    uA = raiz4

elif raiz5>raiz1 and raiz5>raiz2 and raiz5>raiz3 and raiz5>raiz4:
    uA = raiz5

#Incerteza do Padrão

ucP = 0.03/2

#Incerteza da resolução

uRes = 0.01/math.sqrt(12)

#Incerteza do forno

uForno = 0.08/math.sqrt(12)

#Incerteza final

uSoma = ucP**2 + uRes**2 + uForno**2 + uHisterese**2 + uA**2
uComb = math.sqrt(uSoma)
uFinal = uComb*2

#Impressão de incertezas
print(f'Incerteza do Padrão: {ucP}')
print(f'Incerteza da Resolução: {uRes}')
print(f'Incerteza do Forno: {uForno}')
print(f'Incerteza do tipo A: {uA}')
print(f'Incerteza da Histerese: {uHisterese}')

print(f'Incerteza Expandida Final: {uFinal}')


"""
Useful functions for processing SMBL Data
"""
import requests
import sys
import re
import libsbml
import xmltodict
import json

def delete_doubles(arr):
    """
    :param arr: list()
    :return: Given list, without duplicated entries
    """
    arr2 = []
    for element in arr:
        if not arr2.__contains__(element):
            arr2.append(element)
    return arr2

def compare_formulas(formulas, charges=[]):
    """
    compares formulas
    :param charges: charges of molecules as list of integers
    :param charge_hydrogen_balance_accepted: boolean, that indecates, wether differences in the number of H atoms are
    accepted, if the difference is accounted for in the charge
    :param formulas: list of formulas
    :return: True, if all formulas have the same components with the same amount
    <formula> must be string: Upper case character marks new Element  (Mg12Ag2  = [Mg12, Ag2] & MG12AG2 = [M,G12,A,G2])
    """
    # Separate Components of Formula
    formulas_split = []
    for formula in formulas:
        formula_split = []

        # separates the atoms
        for char in formula:
            if char.isupper():
                formula_split.append(char)
            else:
                formula_split[len(formula_split) - 1] = formula_split[len(formula_split) - 1] + char

        # adds "1" to formula, if no number is given at the end
        for i in range(len(formula_split)):
            if re.search("[0-9]", formula_split[i]) is None:
                formula_split[i] = formula_split[i] + "1"

        # adds separated formulas to a list
        formulas_split.append(formula_split)

    # Iterates through all formulas
    for j in range(len(formulas_split) - 1):
        for component in formulas_split[j]:
            # accounts for hydrogen - charge relationship
            if charges and not re.search("^H(?![a-z])+([0-9])*", component) is None:
                component = int(component.split("H")[1])
                component = component + (charges[j + 1] - charges[j])
                component = "H" + str(component)

            # Check next element for current element
            if component not in formulas_split[j + 1]:
                return False

        # Check whether all components were in formula
        if len(formulas_split[j]) != len(formulas_split[j + 1]):
            return False

    return True

# ++ Get Metabolite Data from BiGG Database ++
def bigg_request(_id: str, search_type: str = "metabolites"):
    """
    Requests an entry from the BIGG Database
    :param _id: str e.g. "nh3"
    :param search_type: str e.g. "metabolites"
    :return: decoded .json into dictionary
    """
    custom_request = "http://bigg.ucsd.edu/api/v2/universal/" + search_type + "/" + _id
    req = requests.get(custom_request, headers={"Content-Type": "application/json"})

    if not req.ok:
        req.raise_for_status()
        sys.exit()

    decoded_req = req.json()
    return decoded_req

# ++ Get Metabolite Data from Biocyc Database ++
def biocyc_request(id_org: str, db: str, id_db: str):
    """
    Requests an entry from the BioCyc DB
    :param db: Database e.g. BIGG, SEED,..
    :param id_db: ID from Database e.g. atp, cpd0001
    :param id_org: ID of organism e.g. GCF_000010185
    :return: decoded .json into dictionary
    """
    custom_request = f"https://websvc.biocyc.org/{id_org}/foreignid?ids={db}:{id_db}&fmt=json"
    req = requests.get(custom_request, headers={"Content-Type": "application/json"})

    if not req.ok:
        req.raise_for_status()
        sys.exit()

    try:
        decoded_req = req.json()
    except json.decoder.JSONDecodeError:
        assert id_org != "meta"
        decoded_req = biocyc_request("meta", db, id_db)
    return decoded_req

# KEGG Request Function
def kegg_get(org_id: str, kegg_id: str):
    request_url = f"http://rest.kegg.jp/get/{org_id}:{kegg_id}"
    req = requests.get(request_url).text.split("\n")
    return req

# ++ Get Metabolite Data from BioCyc Database ++
def biocyc_get(id_org: str, id_biocyc: str, detail: str = "full"):
    """
    Requests an entry from the BioCyc DB
    :param detail: either none, low or full, defaults to full
    :param id_biocyc: ID of object e.g. ATP
    :param id_org: ID of organism e.g. GCF_000010185
    :return: decoded .xml into dictionary
    """
    custom_request = f"https://websvc.biocyc.org/getxml?id={id_org}:{id_biocyc}&detail={detail}"
    req = requests.get(custom_request)

    if not req.ok:
        req.raise_for_status()
        sys.exit()

    decoded_req = xmltodict.parse(req.content)
    return decoded_req

def biocyc_get_from_formula(id_org: str, formula: str):
    """
    Requests an entry from the BioCyc DB
    :param formula:
    :param detail: either none, low or full, defaults to full
    :param id_biocyc: ID of object e.g. ATP
    :param id_org: ID of organism e.g. GCF_000010185
    :return: decoded .xml into dictionary
    """
    custom_request = f"https://websvc.biocyc.org/{id_org}/CF?cfs={formula}&fmt=json"
    req = requests.get(custom_request)

    if not req.ok:
        req.raise_for_status()
        sys.exit()

    try:
        decoded_req = req.json()
    except json.decoder.JSONDecodeError:
        assert id_org != "meta"
        decoded_req = biocyc_get_from_formula("meta", formula)
    return decoded_req

def make_cv_term(link: str, qual_type=libsbml.BQB_IS):
    """
    :param qual_type:
    :param link: string that is added to CV-Term
    :return: libsbml.CVTerm
    This method is not generic, but only creates species and reaction standard CV Terms.
    """
    c = libsbml.CVTerm()
    c.setQualifierType(libsbml.BIOLOGICAL_QUALIFIER)
    c.setBiologicalQualifierType(qual_type)
    c.addResource(link)
    return c

def add_link_annotation_species(model, lnk, qual_type, s_id):
    """
    :param qual_type: libsbml.QUALIFIER
    :param model: libsbml.model
    :param lnk: string
    :param s_id: string
    :return: libsbml.model
    """
    cv_term = make_cv_term(lnk, qual_type)

    # eliminate duplicates
    list_cv = []
    for i in range(model.getSpecies(s_id).getNumCVTerms()):
        list_cv.append(model.getSpecies(s_id).getCVTerm(i))

    if cv_term not in list_cv:
        model.getSpecies(s_id).addCVTerm(cv_term)
    return model

def add_link_annotation_reaction(model, lnk, qual_type, s_id):
    """
    :param qual_type: libsbml.QUALIFIER
    :param model: libsbml.model
    :param lnk: string
    :param s_id: string
    :return: libsbml.model
    """
    cv_term = make_cv_term(lnk, qual_type)

    # eliminate duplicates
    list_cv = []
    for k in range(model.getReaction(s_id).getNumCVTerms()):
        list_cv.append(model.getReaction(s_id).getCVTerm(k))

    if cv_term not in list_cv:
        model.getReaction(s_id).addCVTerm(cv_term)
    return model

def add_note_species(model, note: str, fbc_id):
    """
    :param fbc_id: str
    :param model: libsbml.model
    :param note: str
    :return: libsbml.model
    """
    str_note = f"\n
{note}
\n  "
    if not model.getSpecies(fbc_id).isSetNotes():
        model.getSpecies(fbc_id).setNotes(str_note)
    else:
        notes_curent = model.getSpecies(fbc_id).getNotes().toXMLString()
        if note not in notes_curent:
            model.getSpecies(fbc_id).appendNotes(str_note)
    return model

def add_note_gene_product(model, note: str, fbc_id):
    """
    :param fbc_id: str
    :param model: libsbml.model
    :param note: str
    :return: libsbml.model
    """
    str_note = f"\n
{note}
\n  "
    if not model.getPlugin('fbc').getGeneProduct(fbc_id).isSetNotes():
        model.getPlugin('fbc').getGeneProduct(fbc_id).setNotes(str_note)
    else:
        notes_curent = model.getPlugin('fbc').getGeneProduct(fbc_id).getNotes().toXMLString()
        if note not in notes_curent:
            model.getPlugin('fbc').getGeneProduct(fbc_id).appendNotes(str_note)
    return model

def add_note_reaction(model, note: str, fbc_id):
    """
    :param model: libsbml.model
    :param note: str
    :param fbc_id: str
    :return:
    """
    str_note = f"\n
{note}
\n  "
    if not model.getReaction(fbc_id).isSetNotes():
        model.getReaction(fbc_id).setNotes(str_note)
    else:
        notes_curent = model.getReaction(fbc_id).getNotes().toXMLString()
        if note not in notes_curent:
            model.getReaction(fbc_id).appendNotes(str_note)
    return model

def dict_add_overlap_to_list(orig_dict, extend_dict):
    for k, v in extend_dict.items():
        if k not in orig_dict:
            orig_dict[k] = v
        else:
            if hasattr(orig_dict[k], '__iter__') and not isinstance(orig_dict[k], str):
                orig_dict[k] = set(orig_dict[k])
            else:
                orig_dict[k] = {orig_dict[k]}

            if hasattr(v, '__iter__') and not isinstance(v, str):
                orig_dict[k] |= set(v)
            else:
                orig_dict[k] |= {v}
            orig_dict[k] = list(orig_dict[k])
    return orig_dict


"""

NOTE:
    You will have to install the Haskell program find-clumpiness on your machine before running this script.
    For more info, see: https://github.com/GregorySchwartz/find-clumpiness
    Also, this script calls the find-clumpiness program using the terminal via linux commands. The commands may not work if you are on Windows.
    Lastly, this script requires you to make a temp folder, where you will not care if all of the files in that folder get deleted.
    The script uses the temp folder to store the JSON strings of the lineages and clumpiness results, which are deleted after each set of clones is done.

The script then feeds in a list of clones into a function that reformats their trees into a format that can be used by find-clumpiness' "JSON" parser. The end
result of this step produces a file for each clone's tree that can now be used with find-clumpiness.

Next, find-clumpiness is run on the files that were just made, outputting a .txt file for each corresponding file with the results.

Finally, the script builds a dictionary that collects all of the find-clumpiness results by iterating through all of the .txt result files.
Some of the clones produce empty results or do not produce results for the selected criteria, such as clones with only a single node or where the
tissues being compared are only found in a single node. The script then removes all of these empty values from the dictionary so that the end result
has a dictionary with each key (clone) having meaningful results that can be used for plotting. This dictionary is then saved as a .JSON file.

The original find-clumpiness results are in the following format:

    property1,property2,value
    PBMC,PBMC,1.0
    PBMC,MLN,1.0
    MLN,PBMC,1.0
    MLN,MLN,1.0

and this script turns it into:

    results[94] : {'PBMC,PBMC': 1.0, 'PBMC,MLN': 1.0, 'MLN,PBMC': 1.0, 'MLN,MLN': 1.0}

with 94 being the 95th (starts from 0) clumpiness result file. There is 1 file for each clone.

"""

import json
import pickle
import pandas as pd
from pathlib import Path, PureWindowsPath
import os
import re
import subprocess

subjects_dict = {
                2: 'Pt19_R',
                3: 'Pt21_R',
                4: 'Pt23_R',
                8: 'Pt14_R',
                9: 'Pt20_R',
                10: 'Pt17_R',
                17: 'Pt25_R',
                }

def get_clumpiness_by_POD():

    # =============================================================================
    # Load tables
    # =============================================================================
    def load_data(patient_ID, patient_name):
        if patient_name in subjects_dict.values():
            samples_table = pd.read_pickle(r"/path_to_sykes_children_data_folder/samples.pkl")
            metadata_table = pd.read_pickle(r"/path_to_sykes_children_data_folder/sample_metadata.pkl")
            metadata_table['value'] = metadata_table['value'].replace("Colon ", "Colon")
            metadata_table['value'] = metadata_table['value'].replace("MLN_allograft", "AxLN")
            metadata_table['value'] = metadata_table['value'].replace("POD37 44", "POD37")
            ###Change Ileum to Ileum_allograft for Pt21_R POD1145
            POD1145_sample_ids = metadata_table['sample_id'].loc[metadata_table['value'] == 'POD1145']
            metadata_table['value'].loc[metadata_table['sample_id'].isin(POD1145_sample_ids)] =  metadata_table['value'].loc[metadata_table['sample_id'].isin(POD1145_sample_ids)].replace("Ileum", "Ileum_allograft")

            df_of_trees = pd.read_pickle(r"/path_to_sykes_children_data_folder/clones_table/{}_clones_table.pkl".format(patient_ID))
            seq_table = pd.read_pickle(r"/path_to_sykes_children_data_folder/sequences_table/{}_sequences_table.pkl".format(patient_ID))
            seq_table.dropna(subset=["clone_id"], inplace=True)
            seq_table['clone_id'] = seq_table['clone_id'].astype(int)

            # =============================================================================
            # Load rejection samples and remove these samples from data
            # =============================================================================
            path = Path(r"/path_to_sykes_children_data_folder/{}_samples_with_rejection.pkl".format(patient_name))
            pickle_in = open(path,"rb")
            rejection_samples = pickle.load(pickle_in)
            pickle_in.close()
            seq_table = seq_table.loc[~seq_table['sample_id'].isin(rejection_samples)]
            metadata_table = metadata_table.loc[~metadata_table['sample_id'].isin(rejection_samples)]
            samples_table = samples_table.loc[~samples_table['id'].isin(rejection_samples)]

            clone_stats = pd.read_pickle(r"/path_to_sykes_children_data_folder/clone_stats/{}_clone_stats.pkl".format(patient_ID))
            functional_clones = clone_stats['clone_id'].loc[clone_stats['functional'] == 1].unique().tolist()
            seq_table = seq_table.loc[seq_table['clone_id'].isin(functional_clones)]

            subject_samples = samples_table['id'].loc[samples_table['subject_id'] == patient_ID].unique()
            subject_metadata_table = metadata_table.loc[metadata_table['sample_id'].isin(subject_samples)]
            tissues = subject_metadata_table['value'].loc[subject_metadata_table['key'] == 'sample_origin'].unique()
            node_min_clone_dict = pd.read_pickle(r"/path_to_sykes_children_data_folder/{}_clones_per_node_min_using_tree.pkl".format(patient_name))

        return df_of_trees, seq_table, tissues, node_min_clone_dict, rejection_samples

    # =============================================================================
    # POD grouping function to group PODs together that are within 2 days of each other
    # Input is name of patient, such as "Pt19_R"
    # Output is list of lists, with each sublist containing the grouped PODs (or single POD if there are no PODs that are grouped with it)
    # And a second output in the same format, except each sublist is a list of sample_ids that belong to the corresponding POD
    # Example: [[0], [20, 22], [100], [110, 112, 114]], [[sample_ids that belong to 0], [sample_ids that belong to 20, 22], [etc], [etc]]
    # =============================================================================
    def group_PODs(patient_ID, patient_name):
        samples_table = pd.read_pickle(r"/path_to_sykes_children_data_folder/samples.pkl")
        metadata_table = pd.read_pickle(r"/path_to_sykes_children_data_folder/sample_metadata.pkl")
        metadata_table['value'] = metadata_table['value'].replace("POD37 44", 'POD37')
        metadata_table['value'] = metadata_table['value'].replace("Colon ", "Colon")
        metadata_table['value'] = metadata_table['value'].replace("MLN_allograft", "AxLN")
        metadata_table['value'] = metadata_table['value'].replace("POD37 44", "POD37")
        ###Change Ileum to Ileum_allograft for Pt21_R POD1145
        POD1145_sample_ids = metadata_table['sample_id'].loc[metadata_table['value'] == 'POD1145']
        metadata_table['value'].loc[metadata_table['sample_id'].isin(POD1145_sample_ids)] =  metadata_table['value'].loc[metadata_table['sample_id'].isin(POD1145_sample_ids)].replace("Ileum", "Ileum_allograft")

        # =============================================================================
        # Load rejection samples and remove these samples from data
        # =============================================================================
        path = Path(r"/path_to_sykes_children_data_folder/{}_samples_with_rejection.pkl".format(patient_name))
        pickle_in = open(path,"rb")
        rejection_samples = pickle.load(pickle_in)
        pickle_in.close()
        metadata_table = metadata_table.loc[~metadata_table['sample_id'].isin(rejection_samples)]
        samples_table = samples_table.loc[~samples_table['id'].isin(rejection_samples)]

        subject_samples = samples_table['id'].loc[samples_table['subject_id'] == patient_ID].unique()
        metadata_table = metadata_table.loc[metadata_table['sample_id'].isin(subject_samples)]
        pod_list_strings = metadata_table['value'].loc[metadata_table['key'] == 'pod'].unique()
        pod_list = [int(''.join(i for i in x if i.isdigit())) for x in pod_list_strings]
        ###sort both using the int version as the guide
        pod_list, pod_list_strings = (list(t) for t in zip(*sorted(zip(pod_list, pod_list_strings))))

        result = []
        result_samples = []
        current_pod = pod_list[0]
        current_samples = metadata_table['sample_id'].loc[metadata_table['value'] == pod_list_strings[pod_list.index(current_pod)]].unique().tolist()
        temp = []
        temp_samples = []
        for next_pod in pod_list[1:]:
            next_samples = metadata_table['sample_id'].loc[metadata_table['value'] == pod_list_strings[pod_list.index(next_pod)]].unique().tolist()
            if len(temp) == 0:
                temp.append(current_pod)
                for sample in current_samples:
                    temp_samples.append(sample)
            if next_pod - current_pod <= 2:
                temp.append(next_pod)
                for sample in next_samples:
                    temp_samples.append(sample)
            else:
                result.append(temp)
                result_samples.append(temp_samples)
                temp_samples = []
                temp = []
            current_pod = next_pod
            current_samples = next_samples
        if pod_list[-1] not in result[-1]:
            result.append([pod_list[-1]])
            result_samples.append(metadata_table['sample_id'].loc[metadata_table['value'] == pod_list_strings[pod_list.index(pod_list[-1])]].unique().tolist())

        return result, result_samples

    # =============================================================================
    # This function returns two dicts:
    # samples_per_tissue: dict[tissue] : [list of sample_ids in that tissue]
    # tissues_per_sample: dict[sample_id] : [tissue in that sample_id] ###this will be a list with a single element (the tissue)
    # =============================================================================
    def get_tissue_sample_ids(patient_ID, patient_name):

        tissues_per_sample = {}
        samples_per_tissue = {}

        if patient_name in subjects_dict.values():
            samples_table = pd.read_pickle(r"/path_to_sykes_children_data_folder/samples.pkl")
            metadata_table = pd.read_pickle(r"/path_to_sykes_children_data_folder/sample_metadata.pkl")
            subject_samples = samples_table['id'].loc[samples_table['subject_id'] == patient_ID].unique()
            metadata_table = metadata_table.loc[metadata_table['sample_id'].isin(subject_samples)]
            metadata_table['value'] = metadata_table['value'].replace("Colon ", "Colon")
            metadata_table['value'] = metadata_table['value'].replace("MLN_allograft", "AxLN")
            metadata_table['value'] = metadata_table['value'].replace("POD37 44", "POD37")
            ###Change Ileum to Ileum_allograft for Pt21_R POD1145
            POD1145_sample_ids = metadata_table['sample_id'].loc[metadata_table['value'] == 'POD1145']
            metadata_table['value'].loc[metadata_table['sample_id'].isin(POD1145_sample_ids)] =  metadata_table['value'].loc[metadata_table['sample_id'].isin(POD1145_sample_ids)].replace("Ileum", "Ileum_allograft")

            # =============================================================================
            # Load rejection samples and remove these samples from data
            # =============================================================================
            path = Path(r"/path_to_sykes_children_data_folder/{}_samples_with_rejection.pkl".format(patient_name))
            pickle_in = open(path,"rb")
            rejection_samples = pickle.load(pickle_in)
            pickle_in.close()
            metadata_table = metadata_table.loc[~metadata_table['sample_id'].isin(rejection_samples)]
            samples_table = samples_table.loc[~samples_table['id'].isin(rejection_samples)]

            sample_ids = metadata_table['sample_id'].unique()
            tissues = metadata_table['value'].loc[metadata_table['key'] == 'sample_origin'].unique()

            for sample_id in sample_ids:
                tissues_per_sample[sample_id] = list(metadata_table['value'].loc[(metadata_table['key'] == 'sample_origin') & (metadata_table['sample_id'] == sample_id)])[0]
            for tissue in tissues:
                samples_per_tissue[tissue] = list(metadata_table['sample_id'].loc[metadata_table['value'] == tissue].unique())

        return tissues_per_sample, samples_per_tissue

    def get_timepoint_sample_ids(patient_ID, patient_name):

        timepoints_per_sample = {}
        samples_per_timepoint = {}

        if patient_name in subjects_dict.values():
            samples_table = pd.read_pickle(r"/path_to_sykes_children_data_folder/samples.pkl")
            metadata_table = pd.read_pickle(r"/path_to_sykes_children_data_folder/sample_metadata.pkl")

            subject_samples = samples_table['id'].loc[samples_table['subject_id'] == patient_ID].unique()

            metadata_table = metadata_table.loc[metadata_table['sample_id'].isin(subject_samples)]
            metadata_table['value'] = metadata_table['value'].replace("Colon ", "Colon")
            metadata_table['value'] = metadata_table['value'].replace("MLN_allograft", "AxLN")
            metadata_table['value'] = metadata_table['value'].replace("POD37 44", "POD37")
            ###Change Ileum to Ileum_allograft for Pt21_R POD1145
            POD1145_sample_ids = metadata_table['sample_id'].loc[metadata_table['value'] == 'POD1145']
            metadata_table['value'].loc[metadata_table['sample_id'].isin(POD1145_sample_ids)] =  metadata_table['value'].loc[metadata_table['sample_id'].isin(POD1145_sample_ids)].replace("Ileum", "Ileum_allograft")

            # =============================================================================
            # Load rejection samples and remove these samples from data
            # =============================================================================
            path = Path(r"/path_to_sykes_children_data_folder/{}_samples_with_rejection.pkl".format(patient_name))
            pickle_in = open(path,"rb")
            rejection_samples = pickle.load(pickle_in)
            pickle_in.close()
            metadata_table = metadata_table.loc[~metadata_table['sample_id'].isin(rejection_samples)]
            samples_table = samples_table.loc[~samples_table['id'].isin(rejection_samples)]

            sample_ids = metadata_table['sample_id'].unique()
            timepoints = metadata_table['value'].loc[metadata_table['key'] == 'pod'].unique().tolist()
            timepoints_original = metadata_table['value'].loc[metadata_table['key'] == 'pod'].unique().tolist()
            timepoints = [int(''.join(i for i in x if i.isdigit())) for x in timepoints]
            timepoints, timepoints_original = (list(t) for t in zip(*sorted(zip(timepoints, timepoints_original))))

            for sample_id in sample_ids:
                result = list(metadata_table['value'].loc[(metadata_table['key'] == 'pod') & (metadata_table['sample_id'] == sample_id)])[0]
                result = int(''.join(i for i in result if i.isdigit()))
                timepoints_per_sample[sample_id] = result
            for timepoint, timepoint_original in zip(timepoints, timepoints_original):
                samples_per_timepoint[str(timepoint)] = list(metadata_table['sample_id'].loc[metadata_table['value'] == timepoint_original].unique())

        return timepoints_per_sample, samples_per_timepoint

    # =============================================================================
    # Body of script starts here
    # =============================================================================
    for patient_ID, patient_name in subjects_dict.items():

        """
        ###IMPORTANT: BELOW IS THE PATH TO THE TEMP FOLDER WHERE ALL FILES WILL BE DELETED
        """
        ###change directory to the directly where find-clumpiness results will be saved
        temp_folder_where_files_will_be_stored_and_deleted = "/path_to_sykes_children_data_folder/temp"
        os.chdir(temp_folder_where_files_will_be_stored_and_deleted)
        subprocess.run(r'find . -type f -delete', shell=True, check=True, stdout=subprocess.PIPE, universal_newlines=True)

        print("Making list of clones for: " + patient_name)

        df_of_trees, seq_table, tissues, node_min_clone_dict, rejection_samples = load_data(patient_ID, patient_name)
        clones_per_tissue_pairs = pd.read_pickle(r"/path_to_sykes_children_data_folder/Clumpiness/{}_clones_per_tissue_pairs_new_1.pkl".format(patient_name))
        timepoints_per_sample, samples_per_timepoint = get_timepoint_sample_ids(patient_ID, patient_name)
        tissues_per_sample, samples_per_tissue = get_tissue_sample_ids(patient_ID, patient_name)
        pod_groups, _ = group_PODs(patient_ID, patient_name)

        results_1 = {}
        node_min = 3 #No reason to get clumpiness for clones with fewer than 3 nodes because their lineages are not complex enough.
        df_node_min = seq_table.loc[seq_table['clone_id'].isin(node_min_clone_dict[node_min])]
        df_node_min['pod'] = df_node_min['sample_id'].apply(lambda x: timepoints_per_sample[x])
        df_node_min['tissue'] = df_node_min['sample_id'].apply(lambda x: tissues_per_sample[x])

        for pod_group in pod_groups:
            print("Currently analyzing: " + str(pod_group) + " for " + subjects_dict[patient_ID])
            clones_in_pod_group = set(df_node_min['clone_id'].loc[df_node_min['pod'].isin(pod_group)].unique().astype(int))
            tissues_per_pod_group = df_node_min['tissue'].loc[(df_node_min['pod'].isin(pod_group))].unique()
            df_by_pod_and_node_min = df_node_min.loc[df_node_min['clone_id'].isin(clones_in_pod_group)]

            results_1["POD" + str(pod_group)] = {}
            ###Make list of clones_ids to be analyzed
            for tissue1 in range(0, len(tissues) - 1):

                for tissue2 in range(tissue1 + 1, len(tissues)):

                    clones_in_tissue_pair = set(clones_per_tissue_pairs[tissues[tissue1] + "+" + tissues[tissue2]])
                    clones_per_tissue_and_pod_group = clones_in_tissue_pair.intersection(clones_in_pod_group)

                    ###Only add clones that have both tissues at this pod_group
                    list_of_clones_result = []
                    if tissues[tissue1] in tissues_per_pod_group and tissues[tissue2] in tissues_per_pod_group:
                        for clone in clones_per_tissue_and_pod_group:
                            tissues_per_clone_and_pod = df_by_pod_and_node_min['tissue'].loc[(df_by_pod_and_node_min['clone_id'] == clone)].unique()
                            if tissues[tissue1] in tissues_per_clone_and_pod and tissues[tissue2] in tissues_per_clone_and_pod:
                                list_of_clones_result.append(clone)

                    if len(list_of_clones_result) > 0:
                        print("Finished making list of clones. Now making files: " + str(len(list_of_clones_result)) + " total files.")
                        print("POD group: " + str(pod_group) + " Tissues: " + tissues[tissue1] + "+" + tissues[tissue2])
                        results_1["POD" + str(pod_group)][tissues[tissue1] + "+" + tissues[tissue2]] = {}
                        # =============================================================================
                        # Iterate through each tree, reformatting it and making files for find-clumpiness' "JSON" parser
                        # =============================================================================
                        df_of_trees_1 = df_of_trees.loc[df_of_trees['id'].isin(list_of_clones_result)]

                        for clone, clone_id in zip(df_of_trees_1['tree'], df_of_trees_1['id']):
                            def parse(input):
                                result = []
                                for key, value in input.items():
                                    if key == "data":
                                        # test1 = value
                                        result.append(parse_data(value))
                                    if key == "children":
                                        if len(value) > 0:
                                            childrenResult = []
                                            for child in value:
                                                childrenResult.append(parse(child))
                                            result.append(childrenResult)
                                        else:
                                            result.append([])
                                return result

                            def parse_data(data):
                                result = {"nodeID" : "ID", "nodeLabels" : []}
                                seq_ids = data['seq_ids']
                                for key, seq_id in seq_ids.items():
                                    metadata = seq_id['metadata']
                                    if len(metadata) > 0:
                                        metadata_pod = int(''.join(i for i in metadata['pod'] if i.isdigit()))
                                        ###Only count nodes that are in the current pod_group
                                        if metadata_pod in pod_group:
                                            if 'Colon ' in metadata['sample_origin']:
                                                metadata['sample_origin'] = metadata['sample_origin'].replace("Colon ", "Colon")
                                            if 'Ileum' in metadata['sample_origin'] and metadata['pod'] == 'POD1145':
                                                metadata['sample_origin'] = metadata['sample_origin'].replace("Ileum", "Ileum_allograft")
                                            result['nodeLabels'].append(metadata['sample_origin'])
                                return result

                            json_data = json.loads(clone)
                            result = parse(json_data["tree"])

                            path = Path(r"{}/{}.JSON".format(temp_folder_where_files_will_be_stored_and_deleted, clone_id))
                            with open(path, 'w') as outfile:
                                json.dump(result, outfile)

                        print("Finished making files. Now running find-clumpiness.")
                        # =============================================================================
                        # This command runs find-clumpiness from the terminal on the files produced above
                        # =============================================================================
                        subprocess.run(r'for file in *.JSON; do cat ${file%%.*}.JSON | find-clumpiness --format "JSON" -e Majority -o ${file%%.*}.txt; done', shell=True, check=True, stdout=subprocess.PIPE, universal_newlines=True)

                        print("find-clumpiness finished. Now collecting results as a dictionary.")

                        ###this directory contains the find-clumpiness results files
                        directory = temp_folder_where_files_will_be_stored_and_deleted

                        # =============================================================================
                        # Collect the results from the find-clumpiness output files as a dictionary
                        # =============================================================================
                        file_results = {}
                        for entry in os.scandir(directory):
                            if entry.path.endswith(".txt") and entry.is_file():
                                with open(entry) as reader:
                                    file_results[entry.name.split(".")[0]] = {}
                                    line_count = 1
                                    for line in reader.readlines():
                                        line_count += 1
                                        if 'property1,property2,value' not in line:
                                            key_value = line.replace("," + str(re.findall("\d+\.\d+", line)[0]) + '\n', "")
                                            file_results[entry.name.split(".")[0]][key_value] = float(re.findall("\d+\.\d+", line)[0])

                        ###remove empty dictionaries, which can be due to trees with only a single node not returning any values from find-clumpiness or the labels all being in a single node.
                        final_results = file_results.copy()
                        node_removed = 0
                        current_tissues = str(tissues[tissue1] + "," + tissues[tissue2])
                        for key in file_results:
                            if current_tissues not in file_results[key]:
                                del final_results[key]
                                node_removed += 1

                        print("Finished with " + tissues[tissue1] + "+" + tissues[tissue2] + ". Removed " + str(node_removed) + " results where there were no values.")
                        results_1["POD" + str(pod_group)][tissues[tissue1] + "+" + tissues[tissue2]] = final_results

                    print("Now deleting find-clumpiness files to prepare for next iteration.")
                    # =============================================================================
                    # Delete all of the files that were created this iteration via the terminal
                    # =============================================================================
                    subprocess.run(r'find . -type f -delete', shell=True, check=True, stdout=subprocess.PIPE, universal_newlines=True)

        # =============================================================================
        # Save results
        # =============================================================================
        path = Path(r"/path_to_sykes_children_data_folder/{}_clumpiness_POD_Specific.pkl".format(patient_name))
        pickle_out = open(path,"wb")
        pickle.dump(results_1, pickle_out)
        pickle_out.close()

    print("Finished!")

get_clumpiness_by_POD()


import warnings
import argparse

import mlflow
from mlflow.tracking.client import MlflowClient
warnings.filterwarnings("ignore", category=FutureWarning)

from custom_preproc_classes.config.core import config

def parse_bool(to_production: str):
    if to_production == "T":
        return True
    elif to_production == "F":
        return False
    else:
        raise ValueError("to_production must be T or F")

def register_mlflow_model(run_id: str, to_production: str, experiment_name: str):
    """
    This function registers a model from an experiment specific run and sets the stage of the model to "Production".

    Args:
        - run_id: Specific id of the model's run
        - to_production: If True, the stage of the newly registered model is set to 'Production'.

    """

    # ---- Register model ---- #
    model_uri = "runs:/" + run_id + "/model"
    mlflow.set_tracking_uri("sqlite:///mlruns.db")
    model_details = mlflow.register_model(model_uri=model_uri, name=experiment_name)
    # print(*model_details, sep = "\n")

    to_production = parse_bool(to_production)

    # ---- Change stage of registered model to production ---- #
    if to_production:
        client = MlflowClient()
        client.transition_model_version_stage(
            name=model_details.name,
            version=model_details.version,
            stage="Production"
        )

if __name__ == "__main__":

    parser = argparse.ArgumentParser()
    parser.add_argument("-r", "--runid", required=True, type=str)
    parser.add_argument("-to_p", "--to_production", required=True, type=str)
    parser.add_argument("-exn", "--experiment_name", required=True, type=str)
    args = parser.parse_args()
    run_id = args.runid
    to_production = args.to_production
    experiment_name = args.experiment_name

    print(run_id)
    print(to_production)

    register_mlflow_model(run_id=run_id, to_production=to_production, experiment_name=experiment_name)


from django.core.management.base import BaseCommand
from home2_app.models import Client

class Command(BaseCommand):
    help = "edit client  name "

    def add_arguments(self, parser):
        parser.add_argument('name', type=str, help="Client_name")
        parser.add_argument('new_name', type=str, help="New_Client_name")

    def handle(self, *args, **kwargs):
        name = kwargs.get('name')
        new_name = kwargs.get('new_name')
        client = Client.objects.filter(name=name).first()
        if client:
            client.name=new_name
            client.save()
            self.stdout.write(f'edit client{client}')
        self.stdout.write(f' clients {name} in base note found')