from typing import Optional
import asyncio
import datetime as dt

import discord

from utils.lib import utc_now

class TextSession:
    """
    Represents an ongoing text session for a single member.

    Attributes
    ----------
    userid
    guildid
    start_time
    total_messages
    total_words
    total_periods
    this_period_start
    this_period_messages
    this_period_words
    timeout
    """
    __slots__ = (
        'userid', 'guildid',
        'start_time', 'end_time',
        'total_messages', 'total_words', 'total_periods',
        'this_period_start', 'this_period_messages', 'this_period_words',
        'last_message_at', 'timeout_task',
        'finish_callback', 'finish_task', 'finished', 'finished_at',
    )

    # Length of a single period
    # period_length = 5 * 60
    period_length = 10
    timeout_length = 2 * period_length

    # Maximum length of a session
    # session_length = 60 * 60
    session_length = 120

    def __init__(self, userid, guildid, start_time):
        self.userid = userid
        self.guildid = guildid

        self.start_time = start_time
        self.end_time = start_time + dt.timedelta(seconds=self.session_length)

        self.total_messages = 0
        self.total_words = 0
        self.total_periods = 0

        self.this_period_start = start_time
        self.this_period_messages = 0
        self.this_period_words = 0

        self.last_message_at = None
        self.timeout_task = None

        self.finish_callback = None
        self.finish_task = None
        self.finished = asyncio.Event()
        self.finished_at = None

    @property
    def duration(self) -> int:
        if self.start_time is None:
            raise ValueError("Cannot take duration of uninitialised session!")

        end = self.finished_at or utc_now()
        return int((end - self.start_time).total_seconds())

    def __repr__(self):
        return (
            "("
            "{self.__class__.__name__}: "
            "userid={self.userid}, "
            "guildid={self.guildid}, "
            "start_time={self.start_time}, "
            "end_time={self.end_time}, "
            "total_messages={self.total_messages}, "
            "total_words={self.total_words}, "
            "total_periods={self.total_periods}, "
            "last_message_at={self.last_message_at}, "
            "finished_at={self.finished_at}"
            ")"
        ).format(self=self)

    @classmethod
    def from_message(cls, message: discord.Message):
        """
        Instantiate a new TextSession from an initial discord message.

        Does not process the given message.
        """
        if not message.guild:
            raise ValueError("Cannot initialise from message outside of Guild context!")
        self = cls(message.author.id, message.guild.id,  message.created_at)
        return self

    def process(self, message: discord.Message):
        """
        Process a message into the session.
        """
        if not message.guild:
            return

        if (message.author.id != self.userid) or (message.guild.id != self.guildid):
            raise ValueError("Invalid attempt to process message from a different member!")

        # Identify if we need to start a new period
        start = self.this_period_start
        if start is not None and (message.created_at - start).total_seconds() < self.period_length:
            self.this_period_messages += 1
            self.this_period_words += len(message.content.split())
        else:
            self.roll_period()
            self.this_period_start = message.created_at
            self.this_period_messages = 1
            self.this_period_words = len(message.content.split())
        self.last_message_at = message.created_at

        # Update the session expiry
        self._reschedule_timeout(self.last_message_at + dt.timedelta(seconds=self.timeout_length))

    def roll_period(self):
        """
        Add pending stats from the current period, and start a new period.
        """
        if self.this_period_messages:
            self.total_messages += self.this_period_messages
            self.total_words += self.this_period_words
            self.total_periods += 1
        self.this_period_start = None

    async def finish(self):
        """
        Finalise the session and set the finished event. Idempotent.

        Also calls the registered finish callback, if set.
        """
        if self.finished.is_set():
            return

        self.roll_period()
        self.finished_at = self.last_message_at or utc_now()

        self.finished.set()
        if self.finish_callback:
            await self.finish_callback(self)

    async def cancel(self):
        """
        Cancel this session.

        Does not execute the finish_callback.
        """
        ...

    def on_finish(self, callback):
        """
        Register a callback coroutine to be executed when the session finishes.
        """
        self.finish_callback = callback

    async def _timeout(self, diff):
        if diff > 0:
            await asyncio.sleep(diff)
        await asyncio.shield(self.finish())

    def _reschedule_timeout(self, target_time):
        """
        Schedule the finish timeout for the given target time.
        """
        if self.finished.is_set():
            return
        if self.finish_task and not self.finish_task.cancelled():
            self.finish_task.cancel()

        target_time = min(self.end_time, target_time)
        dist = (target_time - utc_now()).total_seconds()
        self.finish_task = asyncio.create_task(self._timeout(dist))


#! /usr/bin/env python
# coding: utf-8

hosts = [
    ['192.168.1.176',22,'root','auqf'],
    ['192.168.1.189',22,'root','auqf']
    ]

service = [
    'yum install -y gcc gcc-c++ libjpeg-devel freetype-devel readline-devel libpng-devel libcurl-devel bzip2-devel openssl-devel libxml2-devel libtool-ltdl-devel',
    'ln -s /usr/lib64/libjpeg.so /usr/lib/libjpeg.so',
    'ln -s /usr/lib64/libpng.so /usr/lib/libpng.so',
    'groupadd www && useradd -g www -s /sbin/nologin -M www',
    'echo "/usr/local/lib" > /etc/ld.so.conf.d/source.conf && ldconfig',
    'cd /usr/local/src/nginx_php/ && tar xzvf pcre.tar.gz      && cd pcre      && ./configure  --libdir=/usr/lib64 --prefix=/usr/local/pcre   && make -j$(nproc) && make install',
    'cd /usr/local/src/nginx_php/ && tar xzvf nginx.tar.gz     && cd nginx     && ./configure  --libdir=/usr/lib64 --prefix=/usr/local/nginx  && make -j$(nproc) && make install',
    'cd /usr/local/src/nginx_php/ && tar xzvf libmcrypt.tar.gz && cd libmcrypt && ./configure  --libdir=/usr/lib64        && make -j$(nproc) && make install',
    'cd /usr/local/src/nginx_php/ && tar xzvf php-src.tar.gz   && cd php-src   && ./buildconf --force && ./configure  --prefix=/usr/local/php \
        --with-gd \
        --with-bz2 \
        --with-zlib \
        --with-curl \
        --with-iconv \
        --with-mhash \
        --enable-zip \
        --enable-fpm \
        --with-mcrypt \
        --enable-soap \
        --with-gettext \
        --without-pear \
        --enable-pcntl \
        --enable-shmop \
        --with-openssl \
        --disable-debug \
        --disable-rpath \
        --enable-shared \
        --enable-bcmath \
        --with-readline \
        --enable-sysvsem \
        --enable-sockets \
        --enable-opcache \
        --with-libxml-dir \
        --enable-mbstring \
        --without-sqlite3 \
        --enable-mbstring \
        --with-png-dir=/usr \
        --without-pdo-sqlite \
        --with-mysql=mysqlnd \
        --with-mysqli=mysqlnd \
        --enable-gd-native-ttf \
        --with-jpeg-dir=/usr/lib \
        --with-freetype-dir=/usr \
        --with-pdo-mysql=mysqlnd \
        --enable-inline-optimization \
        --with-config-file-path=/etc \
        --with-config-file-scan-dir=/etc/php.d \
    && make -j$(nproc) && make install',
    'cd /usr/local/src/nginx_php/ && tar xzvf phpredis.tar.gz    && cd phpredis    && phpize && ./configure && make -j$(nproc) && make install',
    'cd /usr/local/src/nginx_php/ && tar xzvf swoole-src.tar.gz  && cd swoole-src  && phpize && ./configure && make -j$(nproc) && make install',
    'cd /usr/local/src/nginx_php/ && tar xzvf zendopcache.tar.gz && cd zendopcache && phpize && ./configure && make -j$(nproc) && make install',
    '\cp -rf /usr/local/src/nginx_php/php.ini /etc/',
    '\cp -rf /usr/local/src/nginx_php/{nginx.conf,conf.d} /usr/local/nginx/conf',
    '\cp -rf /usr/local/src/nginx_php/status.html /usr/local/nginx/html',
    '\cp -rf /usr/local/src/nginx_php/php-fpm.conf /usr/local/php/etc/',
    '\cp -rf /usr/local/src/nginx_php/php-fpm.service /etc/init.d && chmod a+x /etc/init.d/php-fpm.service'

]

#'cd /usr/local/src/apache_php/ && tar xzvf mhash.tar.gz     && cd mhash     && ./configure && make -j$(nproc) && make install',
#'cd /usr/local/src/apache_php/ && tar xzvf mcrypt.tar.gz    && cd mcrypt    && LD_LIBRARY_PATH=/usr/local/lib && ./configure && make -j$(nproc) && make install',
path = {
        'src_path':'./service_src_packge/apache_php',
        'dst_path':'/usr/local/src'
        }

service_port = 80


from mrjob.job import MRJob
from mrjob.step import MRStep

class T10M(MRJob):
    def mapper_aggregate(self, _, line):
        try:
            fields = line.split(",")
            if len(fields) == 9:
                miner = fields[2]
                size = int(fields[4])
                yield miner, size
        except:
            pass

    def reducer_aggregate(self, key, values):
        yield None, (key, sum(values))

    def reducer_sort(self, _, values):
        sorted_values = sorted(values, reverse=True, key=lambda x: x[1])

        for v in sorted_values[:10]:
            yield v[0], v[1]

    def steps(self):
        return [MRStep(mapper=self.mapper_aggregate,
                       reducer=self.reducer_aggregate),
                MRStep(reducer=self.reducer_sort)]

if __name__ == '__main__':
    T10M.run()
    #job_1648683650522_5938
	#http://andromeda.student.eecs.qmul.ac.uk:8088/proxy/application_1648683650522_5938/


from network import *
from autograd import Tensor
from ops import cross_entropy
from datetime import datetime
import random

# Example of building a LeNet

# ref: https://d2l.ai/chapter_convolutional-neural-networks/lenet.html

# pip install python-mnist

def one_hot(y, n_classes):
    return np.eye(n_classes)[y]

lenet = Sequential(
    LazyConv2d(6, kernel_size=5, padding=2), Sigmoid(),
    AvgPool2d(kernel_size=2, stride=2),
    LazyConv2d(16, kernel_size=5), Sigmoid(),
    AvgPool2d(kernel_size=2, stride=2),
    Flatten(),
    LazyLinear(120), Sigmoid(),
    LazyLinear(84), Sigmoid(),
    LazyLinear(10))

from mnist import MNIST

# download MNIST and unzip it in the data directory
# http://yann.lecun.com/exdb/mnist/
mndata = MNIST('data')
train_images, train_labels = mndata.load_training()

def gradient_descent(x, grad):
    learning_rate = 0.1
    return x - learning_rate * grad

mini_batch = 128
epochs = 5

for epoch in range(epochs):
    print(f'epoch: {epoch}, {datetime.now()}')
    for i in range(0, 59904, mini_batch):
        x = Tensor(np.array(train_images[i:i+mini_batch]).reshape(mini_batch, 1, 28, 28) / 255.0)
        y = Tensor(one_hot(train_labels[i:i+mini_batch], 10))
        y_pred = lenet(x)
        loss = cross_entropy(y_pred, y)
        loss.backward()
        lenet.update_gradient(gradient_descent)
        print(f'{datetime.now()} mini_batch: {i:05}: {loss.ndarray}')

def predict(image):
    print(mndata.display(image))
    x = Tensor(np.array(image).reshape(1, 1, 28, 28) / 255.0)
    y_pred_probabilities = lenet(x)
    result = np.argmax(y_pred_probabilities.ndarray, axis=1)[0]
    print(f'prediction: {result}')
    return result

test_images, test_labels = mndata.load_testing()

print('Examples of prediction')
for i in range(10):
    choice = random.randrange(0, 10000)
    predict(test_images[choice])

print('Validating...(takes around 2 minutes)')
# validate on test set
x = Tensor(np.array(test_images).reshape(10000, 1, 28, 28) / 255.0)
y_pred_probabilities = lenet(x)
predictions = np.argmax(y_pred_probabilities.ndarray, axis=1)

true_values = np.array(test_labels)
N = true_values.shape[0]
accuracy = (true_values == predictions).sum() / N
print(f'validation accuracy: {accuracy}')

print('Examples of wrong prediction')
wrong_predictions = np.where(true_values != predictions)[0]
for i in range(10):
    choice = random.randrange(0, len(wrong_predictions))
    image_index = wrong_predictions[choice]
    predict(test_images[image_index])
    print(f'Actual: {test_labels[image_index]}')


import csv
import glob
import os
import re

def create_gid(year, week, away_team_id, home_team_id):
    return "{0}_{1}_{2}_{3}".format(year, week, away_team_id, home_team_id)

def parse_gid(gid):
    toks = gid.split("_")
    vals = dict()
    vals["year"] = toks[0]
    vals["week"] = toks[1]
    vals["away_team_id"] = toks[2]
    vals["home_team_id"] = toks[3]
    return vals

def create_game_keys(directory, team_ids):
    data_files = glob.glob(os.path.abspath(directory)+'/*games.tsv')
    gids = []

    for filename in data_files:
        matches = re.findall(r'\d{4}', filename)
        year = matches[0]
        reader = csv.DictReader(open(filename), delimiter='\t')

        for row in reader:
            away_team_id = None
            home_team_id = None
            if row["Away"] == "@":
                away_team_id = team_ids[row["Winner/tie"]]
                home_team_id = team_ids[row["Loser/tie"]]
            else:
                home_team_id = team_ids[row["Winner/tie"]]
                away_team_id = team_ids[row["Loser/tie"]]

            week = row["Week"]
            gid = create_gid(year, week, away_team_id, home_team_id)
            gids.append(gid)

    return gids

def feat_score_diff(raw_data):
    score_diff = dict()
    for k,v in raw_data.iteritems():
        score_diff[k] = int(v["ScoreOff"]) - int(v["ScoreDef"])

    return score_diff


from ..models import User, BonusRate, AccountBalance, UserReferral
from ...profiles.models import Notification

class Query:

    _model = None

    def _get_object(self, *args, **kwargs):
        # print("*...", args, type(args))
        # print("**...", kwargs, type(kwargs))
        try:
            return self._model.objects.get(*args, **kwargs)
        except self._model.DoesNotExist:
            return None

    def _get_queryset(self, *args, **kwargs):
        try:
            return self._model.objects.filter(*args, **kwargs)
        except self._model.DoesNotExist:
            return None

    def get_or_none(self, Klass, qs: bool = False, *args, **kwargs):
        self._model = Klass
        if qs:
            return self._get_queryset(*args, **kwargs)
        return self._get_object(*args, **kwargs)

    def save_referred_users(self, user, key):
        """
        :param user: referred User 1
        :param key: referrer User 2
        :return: boolean True / None
        """
        called = self.get_or_none(Klass=User, ref=key)
        print("called.....", called, type(called))
        if key and called:
            referrer = self.get_or_none(Klass=UserReferral, referrer__ref__exact=key)
            referred = self.get_or_none(Klass=UserReferral, qs=True, referrer__phone=user.phone)
            # print("referrer.....", referrer, type(referrer))
            # print("referred.....", referred, type(referred))
            if referrer and referred.exists():
                referred.update(called=called, is_referred=True)
                referrer.referred_users.add(user.pk)
                Notification.objects.create(
                    profile=referrer.refferer.profile,
                    title="Sizning promo-kodingiz orqali yangi foydalanuvchi"
                          "ro'yhattadan o'tdi. Sizga 2 ball qoshildi."
                )
                return True
        return None

    @staticmethod
    def create_or_get_bonus_rate():
        qs = BonusRate.objects.filter(is_active=True)
        if qs.exists():
            return qs.first()
        rate = BonusRate.objects.create(value=1000)
        return rate

    def update_user_balance(self, user):
        is_success = False
        current_user = self.get_or_none(Klass=UserReferral, referrer=user)
        rate = self.create_or_get_bonus_rate()
        if current_user:
            called_user = current_user.called
            if current_user.is_referred and called_user:
                called_user_balance = self.get_or_none(Klass=AccountBalance, user=called_user)
                if called_user_balance:
                    bonus = called_user_balance.bonus + 2
                    called_user_balance.amount = bonus * rate.value
                    called_user_balance.bonus = bonus
                    called_user_balance.save()
                    is_success = True
        current_user_balance = self.get_or_none(Klass=AccountBalance, user=user)
        if current_user_balance:
            bonus = current_user_balance.bonus + 1
            current_user_balance.amount = bonus * rate.value
            current_user_balance.bonus = bonus
            current_user_balance.save()
            is_success = True
        return is_success


from collections import deque

def solution(priorities, location):
    answer = 0
    d = deque(priorities)
    priorities.sort(reverse=True)
    while d:
        while d[0] != priorities[0]:
            d.rotate(-1)
            location -= 1
            if location < 0:
                location += len(priorities)
        answer += 1
        del priorities[0]
        d.popleft()
        if not location:
            return answer
        location -= 1


import requests
import os
from lxml import html
import pickle

balance_sheet_url = 'https://economictimes.indiatimes.com/hdfc-bank-ltd/balancesheet/companyid-{}.cms'
balance_sheet_output_file = os.path.join(os.path.dirname(__file__), 'balance_sheet_data.txt')

bank_nifty = {  #'yes_bank' :'16552',
                # 'icici_bank' :'9194',
                'federal_bank' :'9211',
                'pnb_bank' :'11585',
                'rbl_bank' :'7750',
                'idfc_bank' :'62245',
                'bob_bank' : '12040',
                'kotak_bank' : '12161',
                'hdfc_bank' : '9195',
                'axis_bank' : '9175',
                'sbin' : '11984',
                'indusind' : '9196'}
et_codes = open('et_codes', 'rb')
codes = pickle.load(et_codes)
# print(codes)
output_file = open(balance_sheet_output_file, 'a')

for bank, code in codes.items():
    url = balance_sheet_url.format(code)
    response = requests.get(url)
    tree = html.fromstring(response.text)
    try :
        Data = tree.xpath('//td/text()')
        NetWorth = Data.index('Net Worth');  # print (NetWorth)
        SecuredLoan = Data.index('Secured Loan');
        TotalYears = SecuredLoan - NetWorth
        TotalLiabilities = Data.index('TOTAL LIABILITIES');  # print (TotalLiabilities)
        TotalCurrentAssests = Data.index('Total Current Assets');  # print (TotalCurrentAssests)
        TotalCurrentLiabilities = Data.index('Total Current Liabilities');  # print (TotalCurrentLiabilities)
        NETCURRENTASSETS = Data.index('NET CURRENT ASSETS');  # print (NETCURRENTASSETS)
        TOTALASSETS = Data.index('TOTAL ASSETS(A+B+C+D+E)');  # print (TOTALASSETS)
    except Exception as err:
        print(err)
        continue
    years = ['2019-04-01', '2018-04-01', '2017-04-01', '2016-04-01', '2015-04-01', ]

    detail_list = []
    try :
        for j in range(1, len(years) + 1):
            detail = {
                'year': years[j - 1],
                'stock_name': bank,
                # 'code':self.company_ids[self.i],
                'net_worth': Data[NetWorth + j],
                'total_liabilities': Data[TotalLiabilities + j],
                'total_current_assests': Data[TotalCurrentAssests + j],
                'net_current_assets': Data[NETCURRENTASSETS + j],
                'total_assets': Data[TOTALASSETS + j]
            }
            detail_list.append(detail)
    except Exception as err:
        print("------------->",bank, err)
    output_file.write(str(detail_list))
    output_file.write('\n')

    print(detail_list)


import itertools
import logging
import time
import warnings
from collections import defaultdict, namedtuple
from contextlib import contextmanager
from copy import copy
from typing import Tuple, List, Dict, Any, Union, TYPE_CHECKING
from warnings import warn

import networkx as nx
from networkx import NetworkXNoPath, NodeNotFound
from py2neo.cypher import Cursor

from weaveio.hierarchy import Hierarchy

from .exceptions import AmbiguousPathError, CardinalityError, DisjointPathError, AttributeNameError
from .parser import QueryGraph
from .results import Table, AstropyTable
from ..path_finding import collapse_classes_to_superclasses

if TYPE_CHECKING:
    from .objects import ObjectQuery, Query, AttributeQuery
    from ..data import Data

@contextmanager
def logtime(name):
    start = time.perf_counter()
    yield
    logging.info(f"{name} took {time.perf_counter() - start: .3f} seconds")

def get_groupids(graph, params):
    groupids = []
    names = []
    for name, query in graph.groupbys.items():
        if name in params:
            groupids.append(query._iterate(limit=None, skip=0, distinct=True, no_groups=True))
            names.append(name)
    return groupids, names

def _execute_groups(groupids, names, params):
    for values in itertools.product(*groupids):
        for name, value in zip(names, values):
            params[name] = value
        yield values, params

SplitQuery = namedtuple('SplitQuery', ['index', 'query'])
SplitResult = namedtuple('SplitResult', ['index', 'result'])

class QueryFunctionBase:
    def _precompile(self):
        raise NotImplementedError

    def __call__(self, *args, **kwargs):
        raise NotImplementedError

    def __iter__(self):
        raise NotImplementedError

    def __repr__(self):
        return f'{self.__class__.__name__}()'

class BaseQuery(QueryFunctionBase):
    one_row = False
    one_column = False

    def __new__(cls, *args, **kwargs):
        from fastapi.encoders import ENCODERS_BY_TYPE
        ENCODERS_BY_TYPE[cls] = lambda x: [x._data, x._node]
        return super().__new__(cls)

    def _debug_output(self, skip=0, limit=None, distinct=False, no_cache=False, graph_export_fname=None,
                      exclusive=True):
        n = self._precompile()
        cached_params = n._get_cached_parameters()
        placeholder_params = n._G.dependency_parameters(self._node)
        params = {**placeholder_params, **cached_params}
        c = n._to_cypher(skip, limit, distinct, no_cache)
        params = n._prepare_parameters(c, params)
        if graph_export_fname is not None:
            if exclusive:
                n._G.export(graph_export_fname, n._node)
            else:
                n._G.export(graph_export_fname)
        return '\n'.join(c), self._G.G.nodes[n._node]['ordering'], params

    def raise_error_with_suggestions(self, obj, exception: Exception):
        self._data.autosuggest(obj, self._obj, exception)
        raise exception

    def __repr__(self):
        try:
            return f'<{self.__class__.__name__}({self._previous._obj}-{self._obj})>'
        except AttributeError:
            return f'<{self.__class__.__name__}({self._obj})>'

    def _precompile(self) -> 'BaseQuery':
        return self

    def _prepare_parameters(self, lines, parameters):
        params = {}
        for k, v in parameters.items():
            if any(k in l for l in lines):
                if isinstance(v, AstropyTable):
                    cols = [c for c in v.colnames if any(c in l for l in lines)]
                    assert len(cols), "No columns are used, but table is used. This should not happen"
                    params[k] = v[cols]
                else:
                    params[k] = v
        return params

    def _to_cypher(self, skip, limit, distinct, no_cache=False) -> List[str]:
        """
        returns the cypher lines, cypher parameters, names of columns, expect_one_row, expect_one_column
        """
        lines = self._G.cypher_lines(self._node, no_cache)
        if distinct:
            lines[-1] = 'RETURN DISTINCT'.join(lines[-1].rsplit('RETURN', 1))
        if skip > 0:
            lines.append(f"\nSKIP {skip}")
        if limit is not None:
            lines.append(f"\nLIMIT {limit}")
        return lines

    def _compile(self, skip, limit, distinct) -> Tuple['BaseQuery', List[str]]:
        with logtime('compiling'):
            r = self._precompile()
            return r, r._to_cypher(skip, limit, distinct)

    def _execute_single_query(self, query, cypher, params):
        return query._data.graph.execute(cypher, **{k.replace('$', ''): v for k, v in params.items()})

    def _get_cached_parameters(self):
        params = {}
        for node in self._G.restricted(self._node).nodes:
            params.update(self._G.G.nodes[node].get('cached_params', {}))
        return params

    def _execute(self, skip=0, limit=None, distinct=False, no_groups=False):
        """
        returns a new query (using _precompile) and an iterator over cursors.
        For queries that have not been `split`, this is a single cursor.
        For queries that have been `split`, there will be multiple cursors.
        :param skip: number of rows to skip
        :param limit: number of rows to limit to
        :param distinct: whether to return distinct rows
        :param no_groups: whether to return a single cursor or a generator of cursors
        """
        with logtime('executing'):
            cached_params = self._get_cached_parameters()
            placeholder_params = self._G.dependency_parameters(self._node)
            params = {**placeholder_params, **cached_params}
            if not no_groups and any(v == '<placeholder>' for v in params.values()):
                groupids, names = get_groupids(self._G, params)
                if groupids:
                    return _execute_groups(groupids, names, params), self
            new, lines = self._compile(skip, limit, distinct)
            cypher = '\n'.join(lines)
            params = self._prepare_parameters(lines, params)
            return self._execute_single_query(new, cypher, params), new

    def _iterate_groups(self, generator, query):
        for split_indexes, params in generator:
            copied = copy(query)  # copy the query, so that the parameters are not shared between the cursors
            query._G.G.nodes[copied._node]['cached_params'] = params  # store the groups for when the query is executed
            yield SplitQuery(split_indexes, copied)

    def _iterate(self, skip=0, limit=None, distinct=False, no_groups=False):
        cursor_or_gen, new = self._execute(skip, limit, distinct, no_groups)  # produces either a cursor or a generator of cursors
        if not isinstance(cursor_or_gen, Cursor):
            yield from self._iterate_groups(cursor_or_gen, new)
        else:
            rows = new._data.rowparser.iterate_cursor(cursor_or_gen, new._names, new._is_products, True)
            i = None
            with logtime('total streaming (by iteration)'):
                for i, row in enumerate(rows):
                    yield new._post_process_row(row)
            if limit == i:
                warnings.simplefilter("always")
                warn(f"This query has been capped to {limit} rows but the result is larger than that. "
                     f"Consider using the `limit` parameter to better limit the result or set `limit` to None to get "
                     f"the full result (although this is not recommended).")

    def __iter__(self):
        yield from self._iterate()

    def _to_table(self, skip=0, limit=None, distinct=False) -> Tuple[Table, 'BaseQuery']:
        with logtime('total streaming'):
            cursor_or_gen, new = self._execute(skip, limit, distinct)  # produces either a cursor or a generator of cursors
            if not isinstance(cursor_or_gen, Cursor):
                return [SplitResult(split_indexes, query._to_table(skip, limit, distinct)) for split_indexes, query in self._iterate_groups(cursor_or_gen, new)], new
            return new._data.rowparser.parse_to_table(cursor_or_gen, new._names, new._is_products), new

    def _post_process_table(self, result, squeeze):
        if isinstance(result, SplitResult):
            return result.index, self._post_process_table(result.result, squeeze)
        if not squeeze:
            return result
        if self.one_column:
            result = result[result.colnames[0]].data
        if self.one_row:
            result = result[0]
        return result

    def _post_process_row(self, row):
        if self.one_column:
            return row[row.colnames[0]]
        return row

    def __call__(self, skip=0, limit=1000, distinct=False, squeeze=True, **kwargs):
        tbl, new = self._to_table(skip, limit, distinct)
        if isinstance(tbl, list):
            return tbl
        tbl = new._post_process_table(tbl, squeeze)
        try:
            if not len(tbl):
                return tbl
        except TypeError:
            return tbl  # not a table, just a value, so return it
        if limit == 1 and squeeze:
            return tbl[0]
        try:
            if limit == len(tbl):
                warnings.simplefilter("always")
                warn(f"This query has been capped to {limit} rows but the result is larger than that. "
                     f"Consider using the `limit` parameter to better limit the result or set `limit` to None to get "
                     f"the full result (although this is not recommended).")
        except TypeError:
            pass
        return tbl

    def __init__(self, data: 'Data', G: QueryGraph = None, node=None, previous: Union['Query', 'AttributeQuery', 'ObjectQuery'] = None,
                 obj: str = None, start: 'Query' = None, index_node = None,
                 single=False, names=None, is_products=None, attrs=None, dtype=None, *args, **kwargs) -> None:
        from .objects import ObjectQuery
        self._single = single
        self._data = data
        if G is None:
            self._G = QueryGraph()
        else:
            self._G = G
        if node is None:
            self._node = self._G.start
        else:
            self._node = node
        self._previous = previous
        self.__cypher = None
        self._index_node = index_node
        self._obj = obj
        if previous is not None:
            if self._index_node is None:
                if isinstance(previous, ObjectQuery):
                    self._index_node = previous._node
                elif self._index_node != 'start':
                    self._index_node = previous._index_node
            if obj is None:
                self._obj = previous._obj
        if start is None:
            self._start = self
        else:
            self._start = start
        if self._obj is not None:
            if not self._obj.startswith('_'):  # system name for other purposes
                self._obj = self._normalise_object(self._obj)[0]
        self._names = [] if names is None else names
        self._is_products = [False]*len(self._names) if is_products is None else is_products
        self.attrs = attrs
        self.dtype = dtype

    def _get_object_of(self, maybe_attribute: str) -> Tuple[str, bool, bool]:
        """
        Given a str that might refer to an attribute, return the object that contains that attribute
        and also whether the attribute and the object are singular.
        plurality table:
            inferred obj | attr
                s        |  s      (i.e. run.obid -> run.ob.obid)
                p        |  p      (i.e. ...-> run.l1specs.snrs) - cannot be inferred just from a name
                s        |  p      (i.e. run.obids -> run.ob.obids)
                p        |  s      (i.e. run.snr -> run.l1spec.snr) - is not allowed
        attribute storage scenarios:
            * attribute is contained in this object - return self
            * attribute is contained in exactly one object - return object
            * attribute is contained in more than one object and ...
                * any of those objects don't have the same inherited class - raise AmbiguousError
                * only subclass objects and all subclass objects from a superclass contain the attribute - return the superclass
                * only subclass objects and not all subclass objects from a superclass contain the attribute - return the superclass
        If a singular path exists, use that
        :return: obj, obj_is_singular, attr_is_singular
        """
        single_name = self._data.singular_name(maybe_attribute)
        if self._obj is not None:
            if self._data.class_hierarchies[self._obj] in self._data.factor_hierarchies[single_name]:
                return self._obj, True, self._data.is_singular_name(maybe_attribute)
        want_single = single_name == maybe_attribute
        # try to get singular paths
        hs = {h for h in self._data.factor_hierarchies[single_name] if self._has_path_to_object(h, True)}
        if len(hs) == 0: # if no singular paths then get all paths
            hs = {h for h in self._data.factor_hierarchies[single_name] if self._has_path_to_object(h, want_single)}
        hs = set(collapse_classes_to_superclasses(self._data.hierarchy_graph, hs))
        if len(hs) == 1:
            obj = hs.pop().__name__
        elif not hs:
            raise AmbiguousPathError(f"There is no path to any object that contains `{single_name}`. {self._data.factor_hierarchies[single_name]} are not reachable")
        else:
            raise AmbiguousPathError(f"`{single_name}` is contained within more than one unrelated object {[h.__name__ for h in hs]}. Be specific.")
        if self._obj is None:
            return obj, True, False
        if not self._data.is_factor_name(maybe_attribute):
            raise ValueError(f"{maybe_attribute} is not a valid attribute name")
        _, obj_is_singulars = self._get_path_to_object(obj, False)
        obj_is_singular = len(obj_is_singulars) == 1 and obj_is_singulars[0]
        attr_is_singular = self._data.is_singular_name(maybe_attribute)
        if obj_is_singular and attr_is_singular:
            pass  # run.obid -> run.ob.obid
        elif not obj_is_singular and not attr_is_singular:
            pass
        elif obj_is_singular and not attr_is_singular:
            pass  # run.obids -> run.ob.obids
        else:
            # ob.runid -> Error
            plural_name = self._data.plural_name(obj)
            original = self._data.singular_name(self._obj)
            raise CardinalityError(f"Requested one `{single_name}` from `{original}` "
                                   f"when `{original}` has several `{plural_name}`")
        return obj, obj_is_singular, attr_is_singular

    def _normalise_object(self, obj: str):
        """
        returns the __name__ of the respective class of the obj and whether it is plural/singular.
        If not a class, raise KeyError
        """
        if obj in self._data.class_hierarchies:
            return obj, True
        singular = self._data.singular_name(obj)
        if singular not in self._data.singular_hierarchies:
            raise KeyError(f"{obj} is not a valid object name")
        try:
            h = self._data.plural_hierarchies[obj.lower()]
            singular = False
        except KeyError:
            h = self._data.singular_hierarchies[obj.lower()]
            singular = True
        return h.__name__, singular

    @property
    def _cypher(self):
        if self.__cypher is None:
            self.__cypher = self._G.cypher_lines(self._node)
        return self.__cypher

    @property
    def _result_graph(self):
        return self._G.restricted(self._node)

    def _plot_graph(self, fname):
        return self._G.export(fname, self._node)

    @classmethod
    def _spawn(cls, parent: 'BaseQuery', node, obj=None, index_node=None, single=False, *args, **kwargs):
        return cls(parent._data, parent._G, node, parent, obj, parent._start, index_node, single, *args, **kwargs)

    def _get_path_to_object(self, obj, want_single) -> Tuple[List[str], List[bool]]:
        return self._data.paths_to_hierarchy(self._obj, obj, want_single)

    def _has_path_to_object(self, obj, want_single) -> bool:
        if self._obj is None:
            return True
        try:
            return len(self._get_path_to_object(obj, want_single)[0]) > 0
        except NetworkXNoPath:
            return False

    def _slice(self, slc):
        """
        obj[slice]
        filter, shrinks, destructive
        filter by using HEAD/TAIL/SKIP/LIMIT
        e.g. obs.runs[:10] will return the first 10 runs for each ob (in whatever order they were in)
        you must use query(skip, limit) to request a specific number of rows in total since this is unrelated to the actual query
        """
        raise NotImplementedError

    def _filter_by_mask(self, mask, single=None, split_node=False):
        """
        obj[boolean_filter]
        filter, shrinks, destructive
        filters based on a list of True/False values constructed beforehand, parentage of the booleans must be derived from the obj
        e.g. `ob.l1stackedspectra[ob.l1stackedspectra.camera == 'red']` gives only the red stacks
             `ob.l1stackedspectra[ob.l1singlespectra == 'red']` is invalid since the lists will not be the same size or have the same parentage
        """
        if mask.dtype != 'boolean':
            raise TypeError(f"The predicate in a filter `query[predicate]`, must be of type boolean. {mask} is of type {mask.dtype}. ")
        single = single if single is not None else self._single
        try:
            n = self._G.add_filter(self._node, mask._node, direct=False, force_single=single, split_node=split_node)
        except SyntaxError:
            raise DisjointPathError(f"{self} cannot be filtered by {mask} since `{self._obj}` != `{mask._obj}`."
                                    f"This may be because a nonsensical `wrt` has specified in an aggregation.")
        return self.__class__._spawn(self, n, single=single)

    def _aggregate(self, wrt, string_op, op_name, predicate=False, expected_dtype=None, returns_dtype=None,
                   remove_infs=None, distinct=False, nulls=False):
        if wrt is None:
            wrt = self._start
        try:
            if predicate:
                n = self._G.add_predicate_aggregation(self._node, wrt._node, string_op)
            else:
                n = self._G.add_aggregation(self._node, wrt._node, string_op, remove_infs, expected_dtype, self.dtype, distinct, nulls)
        except SyntaxError:
            raise DisjointPathError(f"Cannot aggregate {self} into {wrt} since they don't share a parent query")
        from .objects import AttributeQuery
        return AttributeQuery._spawn(self, n, wrt._obj, wrt._node, dtype=returns_dtype, single=True, factor_name=op_name)


import json
from json import JSONDecodeError

import requests

from regear_functions import Spreadsheet

def get_guildmember_list():
	respond = requests.get(
		'https://gameinfo.albiononline.com/api/gameinfo/guilds/6g1WhBpsQ_W2eldCMtuTrw/members')
	return respond

def get_last_death(playerId):
	respond = requests.get(
		f'https://gameinfo.albiononline.com/api/gameinfo/players/{playerId}/deaths')
	try:
		outp = respond.json()[0]['EventId']
	except IndexError:
		outp = None
	except JSONDecodeError:
		outp = None
	print(f'Last death : {outp}')
	return outp

def main_writer_json(outp_get_guildmember_list, SpreadsheetVariable):
	try:
		raw_json = outp_get_guildmember_list.json()
		hot_json = {k['Name']: k['Id'] for k in raw_json}
		events_json = {k: get_last_death(hot_json[k]) for k in hot_json.keys()}
	except JSONDecodeError:
		return
	print(f'events_json: {events_json}')

	with open('guild_members.json', 'w') as handler:
		json.dump(hot_json, handler, indent=4)

	with open('guild_events.json', 'r') as handler:
		old_events = json.load(handler)
	print(f'old_events: {old_events}')

	new_events = old_events

	for i in events_json:
		op = events_json[i]
		print(f'Processing {op}')
		try:
			if old_events[i] != events_json[i] and op is not None:
				print(f'{op} IS NOT IN OLD_EVENTS!!! APPENDING IT INTO SPREADSHEET!')
				try:
					SpreadsheetVariable.push(str(op))
				except TimeoutError:
					try:
						SpreadsheetVariable.push(str(op))
					except TimeoutError:
						SpreadsheetVariable.push(str(op))
				new_events[i] = op
				print(new_events)
		except KeyError:
			continue

	with open('guild_events.json', 'w') as handler:
		print('IN ENCLOSING GUILD_EVENTS_JSON')
		json.dump(new_events, handler, indent=4)

if __name__ == "__main__":

	outp_get_guildmember_list = get_guildmember_list()
	raw_json = outp_get_guildmember_list.json()
	hot_json = {k['Name']: k['Id'] for k in raw_json}
	events_json = {k: get_last_death(hot_json[k]) for k in hot_json.keys()}
	with open('guild_events.json', 'w') as handler:
		json.dump(events_json, handler, indent=4)

	spreadsheet = Spreadsheet()
	print(spreadsheet.url())
	while True:
		main_writer_json(get_guildmember_list(), spreadsheet)


import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import models, layers, backend
from tensorflow.keras.datasets import mnist  # datasets

# 데이터 셋 준비
(X_train, y_train), (X_test, y_test) = mnist.load_data()
img_rows, img_cols = X_train.shape[1:] # 이미지 크기
# print(img_rows, img_cols)   # 28, 28

# 데이터의 포멧에 따라, 채널의 위치를 조정
backend.image_data_format()

# 데이터를 이런 형식으로 맟춰서 준비

# 채널 => 흑백이미지(1), 칼라(3) => (60000, 28, 28, 3)
input_channel = 1

if backend.image_data_format == 'channels_last':
    # (1, rows, cols) => (1, H, W)
    # X_train.reshape(빈자리알아서(-1),채널,img_rows, img_cols)
    X_train = X_train.reshape( -1, 1, img_rows, img_cols )  # (60000, 1, 28, 28)
    X_test  = X_test.reshape( -1, 1, img_rows, img_cols )   # (10000, 1, 28, 28)
    input_shape = ( input_channel, img_rows, img_cols )

else :  # 'channels_last'
    X_train = X_train.reshape( -1, img_rows, img_cols, 1 )
    X_test  = X_test.reshape( -1, img_rows, img_cols, 1 )
    input_shape = ( img_rows, img_cols ,input_channel )

# print(X_train.shape, X_test.shape)
# print(X_train.dtype)# 데이터 성분, 범위, 타입

# 타입 변경
X_train = X_train.astype('float32')
X_test  = X_test.astype('float32')
print(X_train.dtype)# 데이터 성분, 범위, 타입

# 정규화
# float32로 변화 처리하지만, 의미는 다르다.
print(y_train[:10])

# 텐서플로우에서 사용한 방식으로 사용해 본다면
# 원-핫 인코딩 스타일로 작성해 본다면 케라스의 유틸리티를 사용
# 답을 구성하는 클래스의 총수( 가짓수, 답안의 케이스 수)
num_classes = 10
y_train = keras.utils.to_categorical(y_train, num_classes)
print(y_train[:10])

# 동일하게 테스트셋에도
y_test = keras.utils.to_categorical(y_test, num_classes)

"""
# 정규화 표준화

- 기본적인 튜토리얼은 각 픽셀값이 0과 1로 정규화 되어있다.
- 정규화 과정을 거치지 않은 상태로
    이파일은 픽셀이 0과 255사이의 값을 가진다.
    그렇기 때문에 전체 픽셀값을 255로 나누어서 정규화를 진행 -> 더욱 빠르게 연산이 가능하다.
"""

# 정규화 처리(최대값 기준)
# -> 이부분 잘모르겠다. 텐서 문법인가...?
X_train /= 255
X_test  /= 255

# print(X_train.shape, y_train.shape, X_test.shape, y_test.shape)

# 신경망 구성하기
# 인공신경망을 만든다고 선언적 표현, 모델 생성
model = models.Sequential()

# 합성곱층 생성 추가
# filters : 출력 채널수 -> 1을 넣어서 32가 출력
# kernal_size : 3x3
model.add( layers.Conv2D(32,
                         kernel_size=(3,3),
                         strides=(1, 1),
                         padding='valid',
                         activation='relu',
                         input_shape=input_shape
                         ) )

# 풀링층 생성 추가
model.add(layers.MaxPool2D(pool_size=(2,2),))
# 과적합 방지 추가
model.add(layers.Dropout(0.25))

# 합성곱층 생성 추가 : 기본값들이 있는 부분들은 생략해서 표현
model.add(layers.Conv2D(64,
                         kernel_size=(3,3),
                         #strides=(1, 1),
                         #padding='valid',
                         activation='relu',
                         #input_shape=input_shape,
                         ))

# 풀링층 생성 추가
model.add( layers.MaxPool2D( pool_size=(2,2) ) )

# 과적합 방지 추가
model.add( layers.Dropout(0.25) )
# 전결합층 추가
model.add(layers.Dense(0.5))

# 출력층 추가 => y
# num_classes => 10
model.add( layers.Dense( num_classes, activation='softmax' ) )
# 손실함수, 최적화, 정확도 추가 혹은 컴파일
model.compile( loss=keras.losses.categorical_crossentropy,
               optimizer='rmsprop',
               metrics=['accuracy']
              )

# 훈련 및 테스트

# 실험값, 임계값
# 실험 횟수, 세대수, 훈련 횟수
epochs     = 10 # 설정값
# 1회 학습시 사용되는 데이터양
batch_size = 128 # 설정값

model.fit( X_train, y_train,
          epochs     = epochs,
          batch_size = batch_size,
          validation_split=0.2
          )

# 테스트 데이터로 확인
score = model.evaluate(X_test, y_test)
print(score)


from __future__ import division
import os
import cv2
import dlib
from part_of_face import Part

class FaceTracking(object):
    """
    얼굴 좌표 인식 후 반환
    """

    def __init__(self):
        self.frame = None

        self.face_left = None
        self.face_right = None
        self.face_top = None
        self.face_bottom = None

        self._face_detector = dlib.get_frontal_face_detector()

        cwd = os.path.abspath(os.path.dirname(__file__))
        model_path = os.path.abspath(os.path.join(cwd, "trained_models/shape_predictor_68_face_landmarks.dat"))
        self._predictor = dlib.shape_predictor(model_path)

    @property
    def face_located(self):
        """Check that the face have been located"""
        try:
            int(self.face_left.x)
            int(self.face_left.y)
            int(self.face_right.x)
            int(self.face_right.y)

            int(self.face_top.x)
            int(self.face_top.y)
            int(self.face_bottom.x)
            int(self.face_bottom.y)
            return True
        except Exception:
            return False

    def refresh(self, frame):
        """Refreshes the frame and analyzes it.

        Arguments:
            frame (numpy.ndarray): 원본 웹캠 영상
        """
        self.frame = frame
        self._analyze()

    # 좌표
    def face_left_coords(self):
        """Returns the coordinates of the left pupil"""
        if self.face_located:
            x = self.face_left.x
            y = self.face_left.y
            return (x, y)

    # 좌표
    def face_right_coords(self):
        """Returns the coordinates of the left pupil"""
        if self.face_located:
            x = self.face_right.x
            y = self.face_right.y
            return (x, y)

    def face_top_coords(self):
        """Returns the coordinates of the left pupil"""
        if self.face_located:
            x = self.face_top.x
            y = self.face_top.y
            return (x, y)

    def face_bottom_coords(self):
        """Returns the coordinates of the left pupil"""
        if self.face_located:
            x = self.face_bottom.x
            y = self.face_bottom.y
            return (x, y)

    def _analyze(self):
        """Detects the face and initialize Eye objects"""
        # 원본 웹캠 frame을 흑백으로 변환
        frame = cv2.cvtColor(self.frame, cv2.COLOR_BGR2GRAY)
        # dlib의 정면 얼굴 검출기
        faces = self._face_detector(frame)

        try:
            landmarks = self._predictor(frame, faces[0])

            self.face_left = Part(landmarks, 0)
            self.face_right = Part(landmarks, 1)
            self.face_top = Part(landmarks, 2)
            self.face_bottom = Part(landmarks, 3)

        except IndexError:
            self.face_left = None
            self.face_right = None
            self.face_top = None
            self.face_bottom = None

    def annotated_frame(self):
        """
        얼굴 point 4개 프레임 반환
        """
        frame = self.frame.copy()

        if self.face_located:
            color = (0, 255, 0)
            face_left_x, face_left_y = self.face_left.x , self.face_left.y
            face_right_x, face_right_y = self.face_right.x , self.face_right.y
            face_top_x, face_top_y = self.face_top.x, self.face_top.y
            face_bottom_x, face_bottom_y = self.face_bottom.x, self.face_bottom.y

            cv2.circle(frame, (face_left_x, face_left_y) , 2,  color)
            cv2.circle(frame, (face_right_x, face_right_y),2,  color)
            cv2.circle(frame, (face_top_x, face_top_y), 2, color)
            cv2.circle(frame, (face_bottom_x, face_bottom_y), 2,  color)

        return frame


import math
def solution(str):
    stk=[]
    for i in str:
        if i=='}':
            if len(stk)!=0:
                d=stk.pop()
                if d=='{':
                    continue  #koi dikkat nahi match ho gaya ab stack se relevant ud bhi jaye toh
                else:          #continue isliye lagaye taki ye '}' wala nahi jud paye and jo pop ho gaya wo ho gaya jane do
                    stk.append('}')
        stk.append(i)
    # print(stk)
    if len(stk)%2==0:
        m=0
        n=0
        for i in stk:
            if i=='{':
                m+=1
            else:
                n+=1
        m=m/2
        n=n/2
        m=math.ceil(m)
        n=math.ceil(n)
        # print(m,n)
        return m+n

    else:
        return -1

if __name__ == '__main__':
    str='}{{}}{{{'
    print(solution(str))

string/count the reversals (bracckets wala).py


#!/usr/bin/python3
"""
CH07. 07_Two_Sum.py
"""

# 입력
nums = [2,7,11,15]
target = 9
# 출력 = [0, 1]

# --------------------------------------------------
def twoSum(nums, target):
    """1. 브루트포스로 모든 경우의수 계산"""
    for i in range(len(nums)):
        for j in range(i+1, len(nums)):
            if nums[i] + nums[j] == target:
                return [i, j]

print(twoSum(nums, target))

# --------------------------------------------------
def twoSum2(nums, target):
    """2. in을 이용한 탐색"""
    for i, n in enumerate(nums):
        complement = target - n

        if complement in nums[i+1:]:
            return [nums.index(n), nums[i+1:].index(complement) + (i+1)]

print(twoSum2(nums, target))

# --------------------------------------------------
def twoSum3(nums, target):
    """3. target-num을 dictionary를 활용해 확인"""
    nums_map = {}
    for i, num in enumerate(nums):
        nums_map[num] = i

    for i, num in enumerate(nums):
        if target - num in nums_map and i != nums_map[target - num]:
            return [i, nums_map[target - num]]

print(twoSum3(nums, target))

# --------------------------------------------------
def twoSum4(nums, target):
    """4. 3을 좀 더 간단하게 축약, 성능상 개선은 미미함."""
    nums_map = {}
    for i, num in enumerate(nums):
        if target - num in nums_map:
            return [nums_map[target - num], i]
        nums_map[num] = i

print(twoSum4(nums, target))

# --------------------------------------------------
def twoSum5(nums, target):
    """이 문제는 nums가 정렬되었다는 보장이 없으므로, 투포인터로 풀 수 없음!"""
    left, right = 0, len(nums) - 1
    while not left == right:
        if nums[left] + nums[right] < target:
            left += 1
        elif nums[left] + nums[right] > target:
            right -= 1
        else:
            return [left, right]

print(twoSum5(nums, target))


from dataclasses import dataclass
from typing import List

import astropy.units as u
import numpy as np
from astropy.coordinates import spherical_to_cartesian

__all__ = ["FieldLines", "FieldLine"]

@dataclass
class FieldLine:
    """
    A single field line.

    Parameters
    ----------
    r : numpy.ndarray
        Radial coordinates.
    lat : numpy.ndarray
        Latitude coordinates **in radians**.
    lon : numpy.ndarray
        Longitude coordinates **in radians**.
    runit : astropy.units.Unit
        Radial coordinate unit.
    """

    r: u.Quantity
    lat: u.Quantity
    lon: u.Quantity

    def __init__(
        self, *, r: np.ndarray, lat: np.ndarray, lon: np.ndarray, runit: u.Unit
    ):
        self.r = r * runit
        self.lat = lat * u.rad
        self.lon = lon * u.rad
        self.runit = runit

    @property
    def xyz(self) -> u.Quantity:
        """
        Cartesian coordinates as a (n, 3) shaped array.
        """
        x, y, z = spherical_to_cartesian(self.r, self.lat, self.lon)
        return np.array([x, y, z]).T * self.r.unit

    @property
    def _rlatlon(self):
        """
        Spherical coordinates as a (n, 3) shaped array.
        """
        return np.column_stack(
            [
                self.r.to_value(self.runit),
                self.lat.to_value(u.rad),
                self.lon.to_value(u.rad),
            ]
        )

@dataclass
class FieldLines:
    """
    A container for multiple field lines.
    """

    flines: List[FieldLine]

    def __init__(self, xs: np.ndarray, runit: u.Unit):
        """
        Parameters
        ----------
        xs : list[numpy.ndarray]
            Field lines. Each array must have lon, lat, r columns in that
            order.
        runit : astropy.units.Unit
            Unit for radial coordinate.
        """
        self.flines = [
            FieldLine(r=x[:, 2], lat=x[:, 1], lon=x[:, 0], runit=runit) for x in xs
        ]
        self.runit = runit

    def __getitem__(self, i):
        return self.flines[i]

    def __len__(self):
        return len(self.flines)

    def __iter__(self):
        for fline in self.flines:
            yield fline

    def save(self, filename):
        """
        Save field lines to file.

        Parameters
        ----------
        filename : pathlib.Path, str
            File to save field lines to.

        Notes
        -----
        Arrays are saved using `numpy.savez_compressed`.
        """
        flines = {f"fline_{i}": fline._rlatlon for i, fline in enumerate(self.flines)}
        flines["runit"] = np.array(self.runit.to_string())
        np.savez_compressed(filename, **flines)

    @classmethod
    def load(cls, filename):
        """
        Load field lines from a file.

        The field lines must have been saved using the ``.save()`` method.

        Parameters
        ----------
        filename : pathlib.Path, str
            File to load field lines from.

        Returns
        -------
        flines : FieldLines
        """
        arrs = np.load(str(filename))
        if "runit" in arrs:
            runit = u.Unit(str(arrs["runit"]))
        else:
            # For backwards compatibility with versions < 0.4, assume
            # solar radii
            runit = u.R_sun

        fline_data = np.array([arrs[k][:, ::-1] for k in arrs if k != "runit"])
        return cls(fline_data, runit=runit)


# from pudb import set_trace; set_trace()
from typing import List

class Solution:
    def smallestNumber(self, num: int) -> int:
        if num < 0:
            return -int(''.join(sorted(str(-num), reverse=True)))
        digits = sorted(str(num))
        for i in range(len(digits)):
            if digits[i] != '0':
                break
        digits[i], digits[0] = digits[0], digits[i]
        return int(''.join(digits))

# sol = Solution()
# tests = [
#     ([3,1,-2,-5,2,-4], [3,-2,1,-5,2,-4]),
#     ([-1,1], [1, -1]),
# ]

# for i, (nums, ans) in enumerate(tests):
#     res = sol.rearrangeArray(nums)
#     if res == ans:
#         print(f'Test {i}: PASS')
#     else:
#         print(f'Test {i}; Fail. Ans: {ans}, Res: {res}')


# -*- coding: utf-8 -*-
"""
Created on Sun Jan 27 19:33:37 2019

@author: myran
"""
import matplotlib.pyplot as plt
from seabird import fCNV
import numpy as np
import sys
import os
import cartopy.crs as ccrs
from shapely.geometry.polygon import LinearRing
import cartopy.feature as cfeature
sys.path.insert(0, '/Users/myran/Documents/Oceanography/Ocean 340')
from Myranda_Dang_a6_340b_cross_sections import twoDemData, calcDistance

def allProfDict(profList, varList):
    '''
    This function creates a "master dictionary" and returns a dictionary containing all the
    stations and their values only during the downcast. Also gets list of max depths of each
    stations
    '''
    ProfDict = {}
    for i in range(len(profList)):
        prof = getDownCastData(profList[i], varList)    #one station
        ProfDict['Station ' + str(i + 1)] = prof
    return(ProfDict)

def getDownCastData(dataset, varList):
    '''
    This function takes in the filepath name of the station's cnv file, creates a dictionary of
    the station variables and values only in the CTD downcasting, and returns that dictionary
    '''
    station = fCNV(dataset)
    newProf = {}
    lat, lon = latLong(station)
    newProf['LATITUDE'] = lat
    newProf['LONGITUDE'] = lon
    for var in station.keys():
        newProf[var] = []
    depth = station['DEPTH']
    maxNum = depth.max()
    end = list(depth).index(maxNum)
    minNum = depth.min()
    start = list(depth).index(minNum)
    for var in varList:    #Look at each variable
        for i in range(start, end):
            newProf[var].append(station[var][i])
    return(newProf)

def latLong(stationDict):
    '''
    This function takes in a dicionary containing a station and its variables and values, and
    returns the latitude and longitude (in that order) of where the profile was taken with rounded
    decimals of four digits after the decimal place
    '''
    head = stationDict.attributes
    lat = float(head['LATITUDE'])
    lat = round(lat, 4)
    long = float(head['LONGITUDE'])
    long = round(long, 4)
    return lat, long

def plotCrossSection(depths2d, dist2d, var2d, figNum, xlabel, ylabel, title, barLabel, maxDepth):
    '''
    This function takes in a 2-D array of depth, distance,
    variable data, title, and a figure number, and create a filled contour plot of the
    variable data array with respect to the depths and distance arrays given
    '''
    plt.figure(num = figNum)
    plt.gca().invert_yaxis()
    plt.ylabel(ylabel)
    plt.xlabel(xlabel)
    plt.title(title, fontsize = 15)
    plt.contourf(dist2d, depths2d, var2d)
    #plt.fill_between(dist2shore,bathy,1000,color='k') #Mapping bathymetry
    cbar = plt.colorbar()
    cbar.set_label(barLabel)
    plt.show()

def plotVar(staDict, varName, figNum, yVar = 'DEPTH'):
    """This function takes in a profile of a variable in the form of a list, another variable
    in a list (most likely depth), the figure number, and the label for the x-axis. It then plots
    the profile on a specific figure, adding labels, a title, inverts the y-axis for proper
    depth profiles, and caption.
    """
    plt.figure(figNum, figsize = (15,20))
    for station in staDict.keys():
        var1 = staDict[station][varName]
        var2 = staDict[station][yVar]
        plt.plot(var1, var2, label = station)
    plt.xlabel(varName)
    plt.ylabel(yVar)
    if 'DEPTH' in yVar:
        plt.gca().invert_yaxis()
        plt.title(varName + ' Profile Plot')
    else:
        plt.title(varName + ' vs '+yVar)
    plt.legend()
    plt.show()
    #Optional saving profiles
    #file = xlabel + '.png'
    #plt.savefig(file)

def mapStations(lats, longs, figSize=(20,20), res='10m', cmap='rainbow'):
    '''
    This function takes in lists of longitude and latitude and plots
    them on a map, as well as taking in optional parameters such as
    figure size, default of (20, 20), map resolution (as string),
    default of '10m', and colormap (string), default of 'rainbow'
    '''
    fig = plt.figure(1,figsize=figSize)
    fig.suptitle('Station Locations', fontsize=25)
    map1 = fig.add_subplot(1,1,1,projection=ccrs.PlateCarree())
    map2 = fig.add_subplot(1,2,1,projection=ccrs.PlateCarree())
    map1.coastlines(resolution=res)
    map1.set_extent([-124, -122, 47, 48.5]) #Puget Sound area
    pslons = [-122.10, -122.10, -122.40, -122.40]
    pslats = [47.89, 48.07, 48.07, 47.89]
    ring = LinearRing(list(zip(pslons, pslats)))
    map2.coastlines(resolution=res)
    map2.set_extent([-122.40, -122.10, 47.89, 48.07], crs=ccrs.PlateCarree())    #Posession Sound
    land = cfeature.NaturalEarthFeature('physical', 'land', res, facecolor='darkseagreen')
    map1.add_feature(land)
    map2.add_feature(land)
    map1.add_geometries([ring],ccrs.PlateCarree(),facecolor='none',edgecolor='darkred',linewidth=4)
    color = eval('plt.cm.'+cmap+'(np.linspace(0,1, len(lats)))')
    for i in range(len(lats)):
        c = color[i]
        lat = lats[i]
        long = longs[i]
        profNum = i+1
        l = "Station" + str(profNum)
        map2.plot(long,lat,'o',color=c, label = l, markersize=14, transform=ccrs.PlateCarree())
    map2.legend()
    map1.set_title('Puget Sound', fontsize=14)
    map2.set_title('Possession Sound', fontsize=14)
    plt.show

def mapDepthCrossSection(lats,longs,varName,stationName,varData,barLabel):
    '''
    This function takes in a list of latitudes and longitudes, the variable name and data that
    are being plotted and shown as changes in each depth layer for each station, label containing
    the units of the inputted variable, and the figure number.

    '''
    plt.figure(figsize=(20,20))
    ax = plt.axes(projection=ccrs.PlateCarree())
    ax.set_title(varName + ' m', fontsize=25)
    land = cfeature.NaturalEarthFeature('physical', 'land', '10m',
                                        facecolor='darkseagreen')
    ax.set_extent([-122.40, -122.10, 47.89, 48.07], crs=ccrs.PlateCarree())
    ax.coastlines(resolution='10m')
    ax.add_feature(land)
    plt.scatter(longs,lats,c=varData,s=200,transform=ccrs.PlateCarree())
    #Need to check to see if SAL/AATT = 0, then pt would be black 'k'
    for i in range(len(lats)):
        lat = lats[i]
        long = longs[i]
        if varData[i] == 0: #Check if station has data or not
            plt.plot(long,lat,'o',color='darkgray', markersize=14, transform=ccrs.PlateCarree())
        plt.text(longs[i], lats[i]+0.005, 'Sta. '+ str(stationName[i]),
                fontsize=14,transform=ccrs.PlateCarree())
    cbar = plt.colorbar()
    cbar.set_label(barLabel)
    plt.show()

def depthRange(depth, station, varName):
    '''
    This function takes in depth(as a string) that samples were collected at, a list of the station,
    their variables and values, and also the variable name of which the user
    wishs to calculate; calculates an average variable over the passed in depth and returns average
    as a float
    '''
    avgList = []
    count = 0
    for i in range(len(station['DEPTH'])):
        num = station['DEPTH'][i]
        if depth in str(num):
            if num < (float(depth) + 2) and num >= (float(depth) - 2):
                value = list(station['DEPTH']).index(num)
                avgList.append(station[varName][value])
                count += 1
    result = 0
    for j in range(len(avgList)):
        result += avgList[j]
    if result != 0 and count != 0:
        result = result/count
    else:
        result = 0
    return result

def createLayersDict(varName, stationDict, layerDepths):
    '''
    This function takes in a variable name, a dictionary containing all the stations and their
    values, and a list of depths that are considered shallow, mid, and deep layers. The function
    creates a dictionary of variable values at those depth layers and returns created dictionary.
    '''
    layerVar = {}
    for layer in layerDepths:
        layerVar[layer] = []
        for station in stationDict.keys():
            if station not in 'Everett Docks':
                layerData = depthRange(str(layer),stationDict[station],varName)
                layerVar[layer].append(layerData)
    return layerVar

def main():
    #Creating list of profile filenames for Stations 1-9
    fp = '/Users/myran/Documents/Oceanography/Ocean Senior Thesis/DATA/rc0015_CTD_MD/PROCESSED/Stations 1-9/'
    fileList = os.listdir(fp)
    profList = []
    for item in fileList:
        if '.cnv' in item:    #Looks to see if file is .cnv file
            openFile = fp + item
            profList.append(openFile)

    #List of variables
    varList = ['PSAL', 'CStarAt0', 'CStarTr0', 'DEPTH', 'flECO-AFL']

    #Attenuation and Salinity labels
    attLabel = 'Attenuation (1/m)'
    salLabel = 'Salinity (PSU)'

    #Create dictionary of all profile
    d = allProfDict(profList, varList)

    #Create list of max depths of all stations
    maxDepth = []
    for station in d.keys():
        maxNum = max(d[station]['DEPTH'])
        maxDepth.append(maxNum)

    #Append Everett Dock file to master dictionary
    dockName = '/Users/myran/Documents/Oceanography/Ocean Senior Thesis/DATA/rc0015_CTD_MD/PROCESSED/RC0015_MD_dockTest.cnv'
    dockFile = getDownCastData(dockName,varList)
    d['Everett Docks'] = dockFile

    # Create numpy array of new depths
    newDepth = np.arange(10, 60, 2)

    #Getting stations that are in a line (Stations 2 to 9) for crossections
    statLine = {}
    stationList = list(d.keys())
    for station in stationList[1:9]:
        statLine[station] = d[station]
    #Create array of latitude and longitude
    lats = []
    longs = []
    for station in d.keys():
        lat = d[station]['LATITUDE']
        lon = d[station]['LONGITUDE']
        lats.append(lat)
        longs.append(lon)

    #Get lats/longs of stations to for transect
    crossLats =[]
    crossLongs = []
    crossDepth = []
    for i in range(4):
        latCross1 = lats[1 + i]
        depth1 = lats[1 + i]
        crossLats.append(latCross1)
        crossDepth.append(depth1)
        longCross1 = longs[1 + i]
        depth2 = longs[1 + i]
        crossLongs.append(longCross1)
        crossDepth.append(depth2)
        latCross2 = lats[8 - i]
        crossLats.append(latCross2)
        longCross2 = longs[8 - i]
        crossLongs.append(longCross2)

    # Get list of distances between datapoints
    dists = calcDistance(crossLats, crossLongs)

    # Create 2-D array of data salinity and attenuation
    attData = twoDemData('CStarAt0', statLine, newDepth)
    salData = twoDemData('PSAL', statLine, newDepth)

    # Create grid of depths and distances
    depths_2d, dists_2d = np.meshgrid(newDepth, dists)

    #Plot lat/long pair of all stations
    mapStations(lats, longs)

    #Plotting cross section
    plotCrossSection(depths_2d,dists_2d,attData,2,'Distance (km)','Depth(m)',
                     'Attenuation CrossSection',attLabel,crossDepth)
    plotCrossSection(depths_2d,dists_2d,salData,3,'Distance (km)','Depth(m)',
                     'Salinity CrossSection', salLabel,crossDepth)

    #Creating "shallow, mid, deep" data of Attenuation and Salinity
    layerDepth = [10,20,30,40,50,60,70,80,90]
    attLayers = createLayersDict('CStarAt0', d, layerDepth)
    salLayers = createLayersDict('PSAL', d, layerDepth)

    #Getting lats/longs for depth layers
    depthCoorLats = lats
    depthCoorLongs = longs
    depthCoorLats.pop()
    depthCoorLongs.pop()

    #Plotting depth interval layers
    count = 4
    for layer in attLayers.keys():
        mapDepthCrossSection(depthCoorLats,depthCoorLongs,'Attenuation: '+str(layer),
                             [1,2,3,4,5,6,7,8,9],np.array(attLayers[layer]),attLabel)
        count += 1
        mapDepthCrossSection(depthCoorLats,depthCoorLongs,'Salinity: '+str(layer),
                             [1,2,3,4,5,6,7,8,9],np.array(salLayers[layer]),salLabel)
        count += 1
    '''
    #Creating depth profiles
    for var in varList: #plotting profiles for each variable
        plotVar(d, var, count)
        count += 1
    plotVar(d, 'PSAL', count, yVar = 'CStarAt0')
    '''

if __name__ == "__main__":
    main()


import time

import RPi.GPIO as GPIO

from thermo import local_settings
from thermo.common.models import *
from thermo.control import thermostat
from thermo.sensor import thermal

def main(available_actions, available_sensors, structs, **kwargs):
    # Assume all sensors are thermal for now
    # Check thermal sensors:
    try:
        thermal.main(
            local_settings.USER_NUMBER,
            local_settings.UNIT_NUMBER,
            available_sensors,
            verbosity=kwargs.get('verbosity', 0),
            validate=kwargs.get('validate', True)
        )
    except Exception as e:
        logging.error('thermal.main error')
        logging.exception(e)

    try:
        clean_old_temps_from_local()
    except Exception as e:
        logging.error('Encountered an error while removing old temperatures from the local db.')
        logging.exception(e)

    for a in available_actions:
        if a.name == 'HEAT':
            # Check thermostat / HVAC:
            try:
                thermostat.main(structs['HVAC'], verbosity=kwargs.get('verbosity', 0))
            except Exception as e:
                logging.error('Uncaught exception in thermostat.main.')
                structs['HVAC'].turn_heat_off()
                logging.exception(e)

@fallback_locally
def setup(local=False, **kwargs):
    # Get all available actions and sensors for this unit

    session = get_session(local=local)
    unit = session.query(Unit).filter(Unit.user == local_settings.USER_NUMBER).filter(
        Unit.id == local_settings.UNIT_NUMBER).all()

    if len(unit) > 1:
        logging.error('Non-unique unit and user combination')
        raise Exception('Non-unique unit and user combination')

    unit = unit[0]

    available_sensors = unit.sensors
    available_actions = unit.actions
    session.close()

    structs = {}
    for i in range(len(available_actions)):
        a = available_actions[i]
        logging.debug('Action %s' % a.name)

        if a.enabled == False:
            logging.debug('Action disabled.')
            del available_actions[i]
            continue

        if a.name == 'HEAT':
            structs['HVAC'] = thermostat.HVAC(a.zone, log=kwargs.get('log', True),
                                              schedule=kwargs.get('schedule', None))
            if kwargs.get('initial', False):
                logging.debug('Testing relays for HVAC.')
                structs['HVAC'].cycle_relays()

    return available_actions, available_sensors, structs

if __name__ == '__main__':
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument('--verbosity', type=int, default=0)
    parser.add_argument('--disable-log', default=True, action='store_false')
    parser.add_argument('--dry-run', default=False, action='store_true')
    parser.add_argument('--sleep', default=10, type=int)
    parser.add_argument('--boot-sleep', default=0, type=int)
    parser.add_argument('--validate', default=1, type=int)

    args = parser.parse_args()

    time.sleep(args.boot_sleep)

    logging.basicConfig(**local_settings.LOGGING)
    verbosity = args.verbosity
    validate = bool(args.validate)
    log = args.disable_log
    dry_run = args.dry_run
    sleep = args.sleep

    # This is done as a precaution to avoid a runaway furnace. In the event that an uncaught exception causes thermoPi
    # to crash, it is possible that the furnace relay will be in the 'on' position. The script, upon restarting, may
    # not be able to resume the thermostat control program. This block ensures that the heat relay is switched off
    # in such a circumstance.
    if 'HEAT' in local_settings.GPIO_PINS:
        GPIO.setmode(local_settings.GPIO_MODE)
        GPIO.setwarnings(False)
        GPIO.setup(local_settings.GPIO_PINS['HEAT'], GPIO.OUT)
        GPIO.setwarnings(True)
        GPIO.output(local_settings.GPIO_PINS['HEAT'], GPIO.LOW)

    logging.debug('Updating available actions and sensors.')

    available_actions, available_sensors, structs = setup(log=log, verbosity=verbosity, initial=True)

    i = 0
    while True:
        i += 1

        if i % (60 / sleep) == 0 or sleep > 60:  # update available sensors and actions every minute

            logging.debug('Updating available actions and sensors.')
            try:
                schedule = structs['HVAC'].schedule  # pass the existing schedule to HVAC if it is set
            except:
                schedule = None  # default, otherwise

            try:
                available_actions, available_sensors, structs = setup(log=log, verbosity=verbosity, schedule=schedule)
            except:
                pass

        main(available_actions, available_sensors, structs, log=log, verbosity=verbosity, validate=validate)
        time.sleep(sleep)


import pytest
import os
from common.read_data import data

# @pytest.fixture(scope="function")
# def testcase_data(request):
#     testcase_name = request.function.__name__
#     return fct_ledger_data.get(testcase_name)

def get_data(yaml_file_name):
    try:
        data_file_path = os.path.join(BASE_PATH, "data", yaml_file_name)
        yaml_data = data.load_yaml(data_file_path)
    except Exception as ex:
        pytest.skip(str(ex))
    else:
        return yaml_data

BASE_PATH =os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
fct_ledger_data = get_data("FctLedgerData.yml")
if __name__ == '__main__':
    print(fct_ledger_data)


# Databricks notebook source
# MAGIC %md-sandbox
# MAGIC
# MAGIC

# MAGIC
# MAGIC

# COMMAND ----------

# MAGIC %md # Keras
# MAGIC
# MAGIC In this notebook, we will build upon the concepts introduced in the previous lab to build a neural network that is more powerful than a simple linear regression model!
# MAGIC
# MAGIC We will use the [California Housing Dataset](https://scikit-learn.org/stable/datasets/real_world.html#california-housing-dataset).
# MAGIC
# MAGIC ## ![Spark Logo Tiny](https://files.training.databricks.com/images/105/logo_spark_tiny.png) In this lesson you:

# MAGIC   - Will modify these parameters for increased model performance:
# MAGIC     - Activation functions
# MAGIC     - Loss functions
# MAGIC     - Optimizer
# MAGIC     - Batch Size
# MAGIC   -  Save and load models

# COMMAND ----------

# MAGIC %run "./Includes/Classroom-Setup"

# COMMAND ----------

# MAGIC %md Let's load in the dataset from Delta.

# COMMAND ----------

train_df = spark.read.format("delta").load(f"{DA.paths.datasets}/california-housing/train")
X_train = train_df.toPandas()
y_train = X_train.pop("label")

test_df = spark.read.format("delta").load(f"{DA.paths.datasets}/california-housing/test")
X_test = test_df.toPandas()
y_test = X_test.pop("label")

# COMMAND ----------

# MAGIC %md-sandbox
# MAGIC
# MAGIC ## Recall from Last Lab
# MAGIC
# MAGIC ##### Steps to build a Keras model
# MAGIC

# COMMAND ----------

# MAGIC %md ## Define a Network
# MAGIC
# MAGIC Let's not just reinvent linear regression. Let's build a model, but with multiple layers using the Sequential model from Keras.
# MAGIC
# MAGIC ![](https://files.training.databricks.com/images/Neural_network.svg)

# COMMAND ----------

# MAGIC %md
# MAGIC ## 1. Activation Function
# MAGIC
# MAGIC If we were to only use a linear combination of linear functions, then we aren't utilizing the power of neural networks!! The power of neural networks derives from the non-linear activation functions.
# MAGIC
# MAGIC **RECAP:** So what are our options for activation functions?

# COMMAND ----------

from tensorflow.keras.layers import Dense
from tensorflow.keras.models import Sequential
import tensorflow as tf
tf.random.set_seed(42)

model = Sequential()

# Input layer
model.add(Dense(20, input_dim=8, activation="relu"))

# Automatically infers the input_dim based on the layer before it
model.add(Dense(20, activation="relu"))

# Output layer
model.add(Dense(1, activation="linear"))

# COMMAND ----------

# MAGIC %md
# MAGIC #### Alternative Keras Model Syntax

# COMMAND ----------

def build_model():
    return Sequential([Dense(20, input_dim=8, activation="relu"),
                       Dense(20, activation="relu"),
                       Dense(1, activation="linear")]) # Keep the last layer as linear because this is a regression problem

# COMMAND ----------

# MAGIC %md
# MAGIC We can check the model definition by calling **`.summary()`**

# COMMAND ----------

model = build_model()
model.summary()

# COMMAND ----------

# MAGIC %md
# MAGIC ## 2. Loss Functions + Metrics
# MAGIC
# MAGIC In Keras, the [*loss function*](https://www.tensorflow.org/api_docs/python/tf/keras/losses) is the function for our optimizer to minimize. *Metrics* are similar to a loss function, except that the results from evaluating a metric are not used when training the model.
# MAGIC
# MAGIC **Recap:** Which loss functions should we use for regression? Classification?

# COMMAND ----------

from tensorflow.keras import metrics
from tensorflow.keras import losses

loss = "mse" # Or loss = losses.mse
metrics = ["mae", "mse"] # Or metrics = [metrics.mae, metrics.mse]

model.compile(optimizer="sgd", loss=loss, metrics=metrics) #sgd stands for stochastic gradient decent
model.fit(X_train, y_train, epochs=10)

# COMMAND ----------

# MAGIC %md
# MAGIC The learning rate is too high for Stochastic Gradient Descent (SGD) - try decreasing the learning rate to a really small value and it won't explode.

# COMMAND ----------

from tensorflow.keras import optimizers

model = build_model()
optimizer = optimizers.SGD(learning_rate=0.000001)
model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
model.fit(X_train, y_train, epochs=10)

# COMMAND ----------

# MAGIC %md
# MAGIC ## 3. Optimizer
# MAGIC
# MAGIC Let's try this again, but using the Adam optimizer. There are a lot of optimizers out there, and here is a great blog post illustrating the various optimizers.
# MAGIC
# MAGIC When in doubt, the Adam optimizer does a very good job. If you want to adjust any of the hyperparameters, you will need to import the optimizer from **`optimizers`** instead of passing in the name as a string.

# COMMAND ----------

from tensorflow.keras import optimizers

model = build_model()
optimizer = optimizers.Adam(learning_rate=0.001)

model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
history = model.fit(X_train, y_train, epochs=20)

# COMMAND ----------

# MAGIC %md
# MAGIC Like we did in the last lesson, we can visualize the loss as a function of the epochs.

# COMMAND ----------

import matplotlib.pyplot as plt

def view_model_loss():
    plt.clf()
    plt.semilogy(history.history["loss"])
    plt.title("Model Loss")
    plt.ylabel("Loss")
    plt.xlabel("Epoch")
    plt.show()

view_model_loss()

# COMMAND ----------

# MAGIC %md
# MAGIC ## 4. Batch Size
# MAGIC
# MAGIC Let's set our **`batch_size`** (how much data to be processed simultaneously by the model) to 64, and increase our **`epochs`** to 20. Mini-batches are often a power of 2, to facilitate memory allocation on GPU (typically between 16 and 512).
# MAGIC
# MAGIC
# MAGIC Also, if you don't want to see all of the intermediate values print out, you can set the **`verbose`** parameter: 0 = silent, 1 = progress bar, 2 = one line per epoch (defaults to 1)

# COMMAND ----------

model = build_model()
model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
history = model.fit(X_train, y_train, epochs=20, batch_size=64, verbose=2)

# COMMAND ----------

# MAGIC %md ## 5. Evaluate
# MAGIC
# MAGIC We can use the `evaluate` function of our model to see how well the model performs on our test dataset.

# COMMAND ----------

model.evaluate(X_test, y_test)

# COMMAND ----------

# MAGIC %md
# MAGIC ## 6. Save Model, Load Model, and Train More
# MAGIC
# MAGIC Whenever you train neural networks, you want to save them. This way, you can reuse them later!
# MAGIC
# MAGIC In our case, we need to save both the architecture and the weights, so we will use **`model.save`**. If you only want to save the weights, you can use **`model.save_weights`**.

# COMMAND ----------

filepath = f"{DA.paths.working_path}/keras_checkpoint_weights.ckpt"
print(filepath)

model.save(filepath)

# COMMAND ----------

# MAGIC %md You can load both the model and architecture together using **`load_model()`**

# COMMAND ----------

from tensorflow.keras.models import load_model

new_model = load_model(filepath)

# COMMAND ----------

# MAGIC %md
# MAGIC Check that the model architecture is the same.

# COMMAND ----------

new_model.summary()

# COMMAND ----------

# MAGIC %md
# MAGIC Let's train it for one more epoch (we need to recompile), and then save those weights.  This is a *warm start.*

# COMMAND ----------

new_model.fit(X_train, y_train, epochs=1, batch_size=64, verbose=2)
new_model.save_weights(filepath) # Using save_weights because the architecture is already saved

# COMMAND ----------

# MAGIC %md
# MAGIC
# MAGIC ## Classroom Cleanup
# MAGIC
# MAGIC Run the following cell to remove lessons-specific assets created during this lesson:

# COMMAND ----------

DA.cleanup()

# COMMAND ----------

# MAGIC %md-sandbox
# MAGIC © 2022 Databricks, Inc. All rights reserved.

# MAGIC Apache, Apache Spark, Spark and the Spark logo are trademarks of the Apache Software Foundation.

# MAGIC

# MAGIC Privacy Policy | Terms of Use | Support

DB/Temp/ml/deep-learning-with-databricks/DL 02 - Keras.py


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import os, logging, torch, json
import os.path as osp
from collections import OrderedDict

from network.base_network import *
from utils import utils
from utils import config as cfg

class Network(BaseNetwork):
    root_logger = logging.getLogger("network %s"%__file__.split('/')[-1])

    def __init__(self, dataloader, args, feat_dim=None):
        super(Network, self).__init__(dataloader, args, feat_dim)

        self.dset = self.dataloader.dataset
        self.attr_embedder = utils.Embedder(args.wordvec, self.dset.attrs, args.data)
        self.emb_dim = self.attr_embedder.emb_dim      # dim of wordvec (attr or obj)

        self.pos_attr_id    = tf.placeholder(tf.int32, shape=[None])
        self.pos_obj_id     = tf.placeholder(tf.int32, shape=[None])
        self.pos_image_feat = tf.placeholder(tf.float32, shape=[None, self.feat_dim])

        self.neg_attr_id    = tf.placeholder(tf.int32, shape=[None])
        self.neg_obj_id     = tf.placeholder(tf.int32, shape=[None])
        self.neg_image_feat = tf.placeholder(tf.float32, shape=[None, self.feat_dim])

        self.test_attr_id   = tf.placeholder(tf.int32, shape=[None], name='test_attr_id')
        self.test_obj_id    = tf.placeholder(tf.int32, shape=[None], name='test_obj_id')

        self.pos_obj_prediction = tf.placeholder(tf.float32, shape=[None, self.num_obj])

        self.lr = tf.placeholder(tf.float32)

    def rep_embedder(self, img, is_training, name):
        """transform ResNet feature to image representation"""
        img = self.MLP(img, self.args.rep_dim, is_training, name, hidden_layers=[])
        return img

    def transformer(self, rep, v_attr, is_training, name):
        """CoN or DecoN in our paper"""

        with tf.variable_scope(name) as scope:
            in_dim = out_dim = self.args.rep_dim

            if not self.args.no_attention:
                attention = self.MLP(v_attr, in_dim, is_training,
                    name='fc_attention', hidden_layers=self.args.fc_att)
                attention = tf.sigmoid(attention)

                rep = attention*rep + rep      # short connection

            hidden = tf.concat([rep, v_attr], axis=1)  # 512+300

            output = self.MLP(hidden, out_dim, is_training,
                name='fc_out', hidden_layers=self.args.fc_compress)

        return output

    def attr_classification(self, emb, is_training, name='classifier'):
        with tf.variable_scope(name) as scope:
            score_A = self.MLP(emb, self.num_attr, is_training, "attr",
                hidden_layers=self.args.fc_cls)
            prob_A = tf.nn.softmax(score_A, 1)
            return score_A, prob_A

    def obj_classification(self, emb, is_training, name='classifier'):
        with tf.variable_scope(name) as scope:
            score_O = self.MLP(emb, self.num_obj, is_training, "obj",
                hidden_layers=self.args.fc_cls)
            prob_O = tf.nn.softmax(score_O, 1)
            return score_O, prob_O

    def RMD_prob(self, feat_plus, feat_minus, repeat_img_feat, is_training, metric):
        """return attribute classification probability with our RMD"""
        # feat_plus, feat_minus:  shape=(bz, #attr, dim_emb)
        # d_plus: distance between feature before&after CoN
        # d_minus: distance between feature before&after DecoN

        d_plus = self.distance_metric(feat_plus, repeat_img_feat)
        d_minus = self.distance_metric(feat_minus, repeat_img_feat)
        d_plus = tf.reshape(d_plus, [-1, self.num_attr]) # bz, #attr
        d_minus = tf.reshape(d_minus, [-1, self.num_attr]) # bz, #attr

        if metric == 'softmax':
            p_plus  = tf.nn.softmax(-d_plus, 1) # (bz, #attr), smaller = better
            p_minus = tf.nn.softmax(d_minus, 1) # (bz, #attr), larger = better
            return p_plus, p_minus

        elif metric == 'rmd':
            d_plus_comp  = self.dset.comp_gamma['b']*d_plus
            d_minus_comp = self.dset.comp_gamma['a']*d_minus
            d_plus_attr  = self.dset.attr_gamma['b']*d_plus
            d_minus_attr = self.dset.attr_gamma['a']*d_minus

            p_comp = tf.nn.softmax(d_minus_comp - d_plus_comp, axis=1)
            p_attr = tf.nn.softmax(d_minus_attr - d_plus_attr, axis=1)

            return p_comp, p_attr

    def build_network(self, test_only=False):
        logger = self.logger('create_train_arch')

        total_losses = []

        batchsize = tf.shape(self.pos_image_feat)[0]

        pos_attr_emb = self.attr_embedder.get_embedding(self.pos_attr_id)
        neg_attr_emb = self.attr_embedder.get_embedding(self.neg_attr_id)

        pos_img = self.rep_embedder(self.pos_image_feat,
            is_training=True, name='embedder') # (bz,dim)
        neg_img = self.rep_embedder(self.neg_image_feat,
            is_training=True, name='embedder') # (bz,dim)

        # rA = remove positive attribute A
        # aA = add positive attribute A
        # rB = remove negative attribute B
        # aB = add negative attribute B
        pos_aA = self.transformer(pos_img, pos_attr_emb, True, name='CoN')
        pos_aB = self.transformer(pos_img, neg_attr_emb, True, name='CoN')
        pos_rA = self.transformer(pos_img, pos_attr_emb, True, name='DeCoN')
        pos_rB = self.transformer(pos_img, neg_attr_emb, True, name='DeCoN')

        attr_emb = self.attr_embedder.get_embedding(np.arange(self.num_attr))
        # (#attr, dim_emb), wordvec of all attributes
        tile_attr_emb = utils.tile_tensor(attr_emb, 0, batchsize)
        # (bz*#attr, dim_emb)

        ########################## classification losses ######################
        # unnecessary to compute cls loss for neg_img

        if self.args.lambda_cls_attr > 0:

            # original image
            _, prob_pos_A = self.attr_classification(pos_img, is_training=True)
            loss_cls_pos_a = self.cross_entropy(
                prob_pos_A, self.pos_attr_id, self.num_attr,
                weight=self.attr_weight)

            # after removing pos_attr
            _, prob_pos_rA_A = self.attr_classification(pos_rA, is_training=True)
            loss_cls_pos_rA_a = self.cross_entropy(
                prob_pos_rA_A, self.pos_attr_id, self.num_attr,
                target=0, weight=self.attr_weight)

            # rmd
            repeat_img_feat = utils.repeat_tensor(pos_img, 0, self.num_attr)
            # (bz*#attr, dim_rep)
            feat_plus  = self.transformer(repeat_img_feat, tile_attr_emb,
                is_training=True, name='CoN')
            feat_minus = self.transformer(repeat_img_feat, tile_attr_emb,
                is_training=True, name='DeCoN')

            prob_RMD_plus, prob_RMD_minus = self.RMD_prob(feat_plus, feat_minus,
                repeat_img_feat, is_training=True, metric=self.args.rmd_metric)

            loss_cls_rmd_plus  = self.cross_entropy(
                prob_RMD_plus, self.pos_attr_id, self.num_attr,
                weight=self.attr_weight)
            loss_cls_rmd_minus = self.cross_entropy(
                prob_RMD_minus, self.pos_attr_id, self.num_attr,
                weight=self.attr_weight)

            loss_cls_attr = self.args.lambda_cls_attr * sum([
                loss_cls_pos_a, loss_cls_pos_rA_a,
                loss_cls_rmd_plus, loss_cls_rmd_minus
            ])

            total_losses.append(loss_cls_attr)

            with tf.device("/cpu:0"):
                tf.summary.scalar('loss/loss_cls_attr', loss_cls_attr)

        if self.args.lambda_cls_obj > 0:
            # original image
            _, prob_pos_O = self.obj_classification(pos_img, is_training=True)
            loss_cls_pos_o = self.cross_entropy(
                prob_pos_O, self.pos_obj_id, self.num_obj,
                weight=self.obj_weight)

            # after removing pos_attr
            _, prob_pos_rA_O = self.obj_classification(pos_rA, is_training=True)
            loss_cls_pos_rA_o = self.cross_entropy(
                prob_pos_rA_O, self.pos_obj_id, self.num_obj,
                weight=self.obj_weight)

            # after adding neg_attr
            _, prob_pos_aB_O = self.obj_classification(pos_aB, is_training=True)
            loss_cls_pos_aB_o = self.cross_entropy(
                prob_pos_aB_O, self.pos_obj_id, self.num_obj,
                weight=self.obj_weight)

            loss_cls_obj = self.args.lambda_cls_obj * sum([
                loss_cls_pos_o,
                loss_cls_pos_rA_o,
                loss_cls_pos_aB_o
            ])

            total_losses.append(loss_cls_obj)

            with tf.device("/cpu:0"):
                tf.summary.scalar('loss/loss_cls_obj', loss_cls_obj)

        ############################# symmetry loss ###########################

        if self.args.lambda_sym > 0:

            loss_sym_pos = self.MSELoss(pos_aA, pos_img)
            loss_sym_neg = self.MSELoss(pos_rB, pos_img)

            loss_sym = self.args.lambda_sym * (loss_sym_pos + loss_sym_neg)
            total_losses.append(loss_sym)

            with tf.device("/cpu:0"):
                tf.summary.scalar('loss/loss_sym', loss_sym)

        ############################## axiom losses ###########################
        if self.args.lambda_axiom > 0:
            loss_clo = loss_inv = loss_com = 0

            # closure
            if not self.args.remove_clo:
                pos_aA_rA = self.transformer(pos_aA, pos_attr_emb,
                    is_training=True, name='DeCoN')
                pos_rB_aB = self.transformer(pos_rB, neg_attr_emb,
                    is_training=True, name='CoN')
                loss_clo = self.MSELoss(pos_aA_rA, pos_rA) + \
                            self.MSELoss(pos_rB_aB, pos_aB)

            # invertibility
            if not self.args.remove_inv:
                pos_rA_aA = self.transformer(pos_rA, pos_attr_emb,
                    is_training=True, name='CoN')
                pos_aB_rB = self.transformer(pos_aB, neg_attr_emb,
                    is_training=True, name='DeCoN')
                loss_inv = self.MSELoss(pos_rA_aA, pos_img) + \
                    self.MSELoss(pos_aB_rB, pos_img)

            # Commutativity
            if not self.args.remove_com:
                pos_aA_rB = self.transformer(pos_aA, neg_attr_emb,
                    is_training=True, name='DeCoN')
                pos_rB_aA = self.transformer(pos_rB, pos_attr_emb,
                    is_training=True, name='CoN')
                loss_com = self.MSELoss(pos_aA_rB, pos_rB_aA)

            loss_axiom = self.args.lambda_axiom * (
                loss_clo + loss_inv + loss_com)
            total_losses.append(loss_axiom)

            with tf.device("/cpu:0"):
                with tf.name_scope('loss'):
                    tf.summary.scalar('loss_axiom', loss_axiom)
                    tf.summary.scalar('loss_clo', loss_clo)
                    tf.summary.scalar('loss_inv', loss_inv)
                    tf.summary.scalar('loss_com', loss_com)

        ############################# triplet loss ############################

        if self.args.lambda_trip > 0:
            pos_triplet = tf.reduce_mean(
                self.triplet_margin_loss(pos_img, pos_aA, pos_rA))
            neg_triplet = tf.reduce_mean(
                self.triplet_margin_loss(pos_img, pos_rB, pos_aB))

            loss_triplet = self.args.lambda_trip * (pos_triplet + neg_triplet)
            total_losses.append(loss_triplet)

            with tf.device("/cpu:0"):
                tf.summary.scalar('loss/loss_triplet', loss_triplet)

        ################################ testing ##############################
        pos_img = self.rep_embedder(self.pos_image_feat, False, "embedder")
        repeat_img_feat = utils.repeat_tensor(pos_img, 0, self.num_attr)
        # (bz*#attr, dim_rep)

        feat_plus = self.transformer(repeat_img_feat, tile_attr_emb,
            is_training=False, name='CoN')
        feat_minus = self.transformer(repeat_img_feat, tile_attr_emb,
            is_training=False, name='DeCoN')

        prob_A_rmd, prob_A_attr = self.RMD_prob(feat_plus, feat_minus,
            repeat_img_feat, is_training=False, metric='rmd')

        _, prob_A_fc = self.attr_classification(pos_img, is_training=False)
        _, prob_O_fc = self.obj_classification(pos_img, is_training=False)

        if self.args.obj_pred is None:
            prob_O = prob_O_fc
        else:
            prob_O = self.pos_obj_prediction

        test_a_onehot = tf.one_hot(self.test_attr_id, depth=self.num_attr, axis=1)
        test_o_onehot = tf.one_hot(self.test_obj_id, depth=self.num_obj, axis=1)
        # (n_p, n_o)
        prob_P_rmd = tf.multiply(
            tf.matmul(prob_A_rmd, tf.transpose(test_a_onehot)),
            tf.matmul(prob_O, tf.transpose(test_o_onehot))
        )

        score_res = dict([
            ("score_rmd",   [prob_P_rmd, prob_A_attr, prob_O]),
        ])

        ############################### summary ###############################

        loss = sum(total_losses)

        with tf.device("/cpu:0"):
            tf.summary.scalar('loss_total', loss)
            tf.summary.scalar('lr', self.lr)

        train_summary_op = tf.summary.merge_all()

        return loss, score_res, train_summary_op

    def train_step(self, sess, blobs, lr, train_op, train_summary_op):

        summary, _ = sess.run(
            [train_summary_op, train_op],
            feed_dict={
                self.pos_attr_id     : blobs[1],
                self.pos_obj_id      : blobs[2],
                self.pos_image_feat  : blobs[4],
                self.neg_attr_id     : blobs[6],
                self.neg_image_feat  : blobs[9],
                self.lr: lr,
            })

        return summary


# Sergii Logosha, 26 Jun 2022

# Program prompts user to input the radius and calculates the area of the circle and the volume of the sphere

import math

# Prompt user to insert the radius
r = float(input("Type in the radius, please (mm): "))

# Calculating and printing out the area of the circle
area = round(math.pi * r ** 2, 2)
print(f"The area of the circle is {area} mm2.")

# Calculating the volume of the sphere
volume = round((4 / 3) * math.pi * r ** 3, 2)
print("The volume of the sphere is {} mm3.".format(volume))


#!/usr/local/bin/python

#python Version 2.7.2
import numpy as np

class Camera:
    def __init__(self, cameraFile):
        #Unpack the config file
        paramArr, valArr = np.loadtxt(cameraFile, dtype=np.str, unpack=True, usecols=[0,2])
        dict = {}
        for i in range(len(paramArr)):
            dict[paramArr[i]] = valArr[i]

        # ***** Public Variables *****
        self.opticalCouplingToFP = self.__float(dict['OpticalCouplingToFP'])
        self.fnumber = self.__float(dict['Fnumber'])
        self.Tb = self.__float(dict['BathTemp'])

    #***** Private functions *****
    def __float(self, val):
        try:
            return float(val)
        except:
            return str(val)


# _merge_reduce_peakset_module.py

__module_name__ = "_merge_reduce_peakset_module.py"
__author__ = ", ".join(["Michael E. Vinyard"])
__email__ = ", ".join(["vinyard@g.harvard.edu",])

# local imports #
# ------------- #
from ._supporting_funcs._load_data import _load_data
from ._supporting_funcs._get_peak_superset import _get_peak_superset
from ._supporting_funcs._unify_peak_matrices import _unify_peak_matrices
from ._supporting_funcs._write_DataDict_mtx_to_h5 import _write_DataDict_mtx_to_h5
from ._supporting_funcs._calculate_master_merged_peaks import _calculate_master_merged_peaks

class _PeakSet:

    def __init__(self, out="./"):

        """Instantiate"""

        self.out = out

    def load_10x(self, group_path, super_dir, minor_dir_10x="outs", name=""):

        """
        Load samples from 10x outs.

        Parameters:
        -----------
        group_path
            Path to directory containing all samples
            type: str

        super_dir
            Local direoctory containing all samples.
            type: str

        minor_dir_10x
            Directory within each sample containing the filtered_barcode_matrix
            default: "outs"
            type: str

        Returns:
        --------
        None

        Notes:
        ------
        """

        self.DataDict = _load_data(group_path=group_path,
                                   super_dir=super_dir,
                                   minor_dir_10x=minor_dir_10x,
                                   name=name)

    def aggregate_peaks(self):

        """
        Aggregate peaks across all samples.

        Parameters:
        -----------
        None

        Returns:
        --------
        None

        Notes:
        ------
        """

        self.aggregate_peak_df = _get_peak_superset(self.DataDict)

    def define_reduced_merged_peaks(self, out=False):

        """
        Defines a new unified peakset across all samples.

        Parameters:
        -----------
        out

        Returns:
        --------
        Saves new peakset to .csv in `out`.

        Notes:
        ------
        """
        if out:
            self.out = out

        self.DataDict = _unify_peak_matrices(self.DataDict, self.aggregate_peak_df, self.out)

    def write(self, out=False):

        """
        Write updated data matrices (with merged peaks) to .h5 files, for each sample.

        Parameters:
        -----------
        out

        Returns:
        --------

        Notes:
        ------
        """

        if out:
            self.out = out

        _write_DataDict_mtx_to_h5(self.DataDict, self.out)


###################################
# tellurium setup script
#
# develop install via
# pip install -e .
###################################
from setuptools import setup  # allows 'python setup.py develop'
import os
try:
    with open(os.path.join(os.path.dirname(__file__), 'VERSION.txt'), 'r') as f:
        version = f.read().rstrip()
except IOError:
    # fallback location
    with open(os.path.join(os.path.dirname(__file__), 'tellurium/VERSION.txt'), 'r') as f:
        version = f.read().rstrip()

setup(name='tellurium',
      version=version,
      author='J. Kyle Medley, Kiri Choi, Matthias König, Lucian Smith, Herbert M. Sauro',
      description='Tellurium: An biological modeling environment for Python',
      url='http://tellurium.analogmachine.org/',
      packages=[
          'tellurium',
          'tellurium.analysis',
          'tellurium.teio',
          'tellurium.sedml',
          'tellurium.notebooks',
          'tellurium.optimization',
          'tellurium.visualization',
          'tellurium.sedml',
          'tellurium.tests',
      ],
      package_data={
          "tellurium": ["*.txt"],
          "tellurium.sedml": ["templates/*.template"],
      },
      install_requires=['pandas>=0.19.2', 'matplotlib>=2.0.0', 'appdirs>=1.4.3']
      )