from pathlib import Path
import os
import sys
from sklearn.tree import DecisionTreeClassifier
from sklearn import metrics
import HW1
import re
import random

def getAllSmaliFiles(dirName):
    # create a list of file and sub directories
    # names in the given directory
    listOfFile = os.listdir(dirName)
    allFiles = list()
    # Iterate over all the entries
    listOfFile = listOfFile[:55] + listOfFile[:-55]
    for entry in listOfFile:
        # Create full path
        fullPath = os.path.join(dirName, entry)
        # If entry is a directory then get the list of files in this directory
        if os.path.isdir(fullPath):
            allFiles = allFiles + getAllSmaliFiles(fullPath)
        else:
            allFiles.append(fullPath)
    allFiles = [i for i in allFiles if i.split(".")[-1] == "smali"]
    return allFiles

def prepare_data(test_flag):
    if test_flag == 1:
        return getAllSmaliFiles("testdata")
    return getAllSmaliFiles("data")

def makeBaselineFeatures(smali_lst):
    app_features = {}
    app_genre = {}
    for i in smali_lst:
        temp = i.split("/")
        #temp[2] is the app name, temp[1] is the genre
        if temp[2] not in app_features:
            app_features[temp[2]] = {'api_calls': {}, 'lib_calls': {}, }
            app_genre[temp[2]] = temp[1]
        txt = Path(i).read_text(errors = 'ignore')
        api_calls = re.findall(r'L.+->.+\(', txt)
        for api in api_calls:
            if api not in app_features[temp[2]]['api_calls']:
                app_features[temp[2]]['api_calls'][api] = 0
            app_features[temp[2]]['api_calls'][api] += 1
            end = api.index("->")
            lib = api[:end+1]
            if lib not in app_features[temp[2]]["lib_calls"]:
                app_features[temp[2]]["lib_calls"][lib] = 0
            app_features[temp[2]]["lib_calls"][lib] += 1
    x = []
    y = []
    common_api = []
    common_lib = []
    for i in app_genre:
        y.append(i)
        api_count = list(app_features[i]["api_calls"].items())
        lib_count = list(app_features[i]["lib_calls"].items())

        mode_api = max(api_count, key=lambda x:x[1])[0]
        mode_lib = max(lib_count, key=lambda x:x[1])[0]

        if mode_api not in common_api:
            common_api.append(mode_api)

        if mode_lib not in common_lib:
            common_lib.append(mode_lib)

        api_index = common_api.index(mode_api)
        lib_index = common_lib.index(mode_lib)

        degree = len(app_features[i]["api_calls"])
        x.append([api_index, lib_index, degree])
    return x, y

def baseline(x, y):
    y_test = y[:1] + y[-1:]
    x_test = x[:1] + x[-1:]

    x_train = x[1:-1]
    y_train = y[1:-1]

    dtc = DecisionTreeClassifier()
    dtc.fit(x_train, y_train)
    dtc_preds = dtc.predict(x_train)
    print("Train Accuracy: ", metrics.accuracy_score(y_train, dtc_preds))
    dtc_preds = dtc.predict(x_test)
    print("Test Accuracy: ", metrics.accuracy_score(y_test, dtc_preds))
    return

def getTestData():
    if os.path.exists("testdata") == False:
        os.mkdir("testdata")
    for i in ["productivity", "entertainment"]:
        if i == "malware":
            continue
        if os.path.exists("testdata/" + i) == False:
            os.mkdir("testdata/" + i)
    gz_list = HW1.parse_test(["productivity", "entertainment"] , "https://apkpure.com/sitemap.xml")
    sample_size = 4
    count = 0
    while count < sample_size:
        category = "productivity"
        if count >= (sample_size/2):
            category = "entertainment"
        cat_gz_lst = [i for i in gz_list if category in i] #small list of gz
        gz_link = random.choice(cat_gz_lst) #one gz
        gz_name = HW1.download_gz(gz_link) #unzipped gz
        #print(gz_name)
        app_urls = HW1.get_urls_from_gz(gz_name) #list of app urls
        link = ''
        app_name = ''
        while link == '' or "https://" not in link:
            app_url = random.choice(app_urls) #page of an app
            link, app_name = HW1.get_dl_link(app_url) #app_name is name of app
        apk_name = HW1.download_app(link, app_name) #app_name + .apk
        #print(apk_name)
        #current_direc = os.getcwd()

        if len(app_name) > 20:
            app_name = app_name[:10]
        directory = "testdata/" + category + "/" + app_name
        if os.path.exists(directory) == False:
            os.mkdir(directory)
        HW1.get_smali(apk_name)#, directory)
        os.system("mv " + app_name + "/smali " + directory)
        os.system("rm -rf " + app_name)
        count = count + 1
    return


import numpy as np
import matplotlib.pyplot as plt

from deepnetwork.core import NeuralNetwork, DenseLayer, Conv2DLayer, FlattenLayer, MaxPoolingLayer
from deepnetwork.optimizers import Adam
from deepnetwork.load import load

def make_model():
    """
    Function to make model
    """
    conv = Conv2DLayer(4, filter_size=3, padding=True, activation='relu', stride=1)
    pool = MaxPoolingLayer()
    conv2 = Conv2DLayer(8, filter_size=3, padding=True, activation='relu', stride=1)
    pool2 = MaxPoolingLayer()
    flatten = FlattenLayer()
    h_1 = DenseLayer(128, 'relu')
    end_network = DenseLayer(11, 'softmax')
    network = NeuralNetwork(conv, pool, conv2, pool2, flatten, h_1, end_network)
    network.optimizer = Adam(beta1=0.90, beta2=0.999, learning_rate=0.004)
    return network

X_train, y_train, X_val, y_val = load(reshape=False)

epochs = 200
batch_size = 512

nn = make_model()

train_errors, valid_errors = nn.fit(data=X_train, labels=y_train,
                                    count_epochs=epochs, validation_set=(X_val, y_val), size_of_batch=batch_size)

plt.plot(train_errors, label='train')
plt.plot(valid_errors, label='test')
plt.legend()
plt.savefig('graph_best.png')


"""
Project 5: Knapsack Approximation Algorithm
This program is the implemetation of Knapsack Approximation Algorithm (ordered items by value/weight).

Author: Xiang Shan Tan

run:
python ./valueweight.py
"""

import numpy as np

# greedy algorithm
# pick the largest value per weight
def ValueWeight(n, values, weights, capacity) :
  # initialize a weight counter
  # this is be used against capacity
  curweight = 0
  curvalue = 0
  # initialize an array to store values of values/weights
  ratios = []
  # initialize empty arrays to store picked values and weights
  valueset = []
  weightset = []

  for i in range(n) :
    if values[i] > 0 :
      ratios.append(values[i] / weights[i])
  #print(ratios)
  # get the index by sorting values from highest to lowest
  sorted = np.array(np.argsort(ratios))[::-1]
  print(f"Sorted Array: {sorted}")

  for i in (sorted):
      temp = curweight + weights[i]
      if temp <= capacity :
          valueset.append(values[i])
          weightset.append(weights[i])
          curvalue += values[i]
          curweight = temp

  return curvalue, curweight, sorted

def main() :
    # total capacity
    capacity = 10
    # highest value
    val = 10
    # total items
    items = 5

    count = 1
    for i in range(count) :
        weights = np.random.randint(1, capacity + 1, size = items)
        values = np.random.randint(1, val + 1, size = items)
        print(f"Weights: {weights}")
        print(f"Values: {values}")

        value, weight, itemset = ValueWeight(items, values, weights, capacity)
        print(value, weight)

if __name__ == "__main__":
    main()


# pip install pyjwt

'''
检查用户的认证模式，同时认证完成后验证用户是否有权限操作
兼容session认证的方式，同时支持JWT，并且两种验证共用同一套权限系统
'''

from django.conf import settings
from django.http import JsonResponse
from django.contrib.auth import get_user_model
from django.core.exceptions import PermissionDenied

import jwt

UserModel = get_user_model()

'''检查用户的认证模式，同时认证完成后验证用户是否有权限操作'''
def auth_permission_required(perm):
    '''检查用户的认证模式，同时认证完成后验证用户是否有权限操作'''
    def decorator(view_func):
        def _wrapped_view(request, *args, **kwargs):
            # 格式化权限
            perms = (perm,) if isinstance(perm, str) else perm

            if request.user.is_authenticated:  # session 验证
                # 正常登录用户判断是否有权限
                if not request.user.has_perms(perms):  # 检查用户权限
                    raise PermissionDenied
            else:
                try:
                    auth = request.META.get('HTTP_AUTHORIZATION').split()  # 获取 token (jwt)  {'Authorization': 'Token '+token})
                except AttributeError:
                    return JsonResponse({"code": 401, "message": "No authenticate header"})

                # 用户通过API获取数据验证流程
                if auth[0].lower() == 'token':
                    try:
                        dict = jwt.decode(auth[1], settings.SECRET_KEY, algorithms=['HS256'])
                        username = dict.get('data').get('username')
                    except jwt.ExpiredSignatureError:  # 过期
                        return JsonResponse({"status_code": 401, "message": "Token expired"})
                    except jwt.InvalidTokenError:  # 非法
                        return JsonResponse({"status_code": 401, "message": "Invalid token"})
                    except Exception as e:  # 载荷信息不存在
                        return JsonResponse({"status_code": 401, "message": "Can not get user object"})

                    try:
                        user = UserModel.objects.get(username=username)  # 检查用户是否存在
                    except UserModel.DoesNotExist:
                        return JsonResponse({"status_code": 401, "message": "User Does not exist"})

                    if not user.is_active:  # 检查用户是否有效
                        return JsonResponse({"status_code": 401, "message": "User inactive or deleted"})

                    # Token登录的用户判断是否有权限
                    if not user.has_perms(perms):  # 检查用户权限
                        return JsonResponse({"status_code": 403, "message": "PermissionDenied"})
                else:
                    return JsonResponse({"status_code": 401, "message": "Not support auth type"})  # 没有找到 session 和 token

            return view_func(request, *args, **kwargs)
        return _wrapped_view
    return decorator

# usage
@auth_permission_required('app_name.select_user')
def user(request):
    if request.method == 'GET':
        _jsondata = {
            "user": "ops-coffee",
            "site": "https://ops-coffee.cn"
        }
        return JsonResponse({"state": 1, "message": _jsondata})
    else:
        return JsonResponse({"state": 0, "message": "Request method 'POST' not supported"})

# 给 User model 添加一个token的静态方法来生成用户Token
import datetime
from django.db import models
from django.conf import settings
class User(models.Model):
    ...

    @property
    def token(self):
        return self._generate_jwt_token()

    def _generate_jwt_token(self):
        '''生成token'''
        headers = {"type":"jwt", "alg":"HS256"}
        payload = {
            'exp': datetime.datetime.now() +  datetime.timedelta(days=1),  # 过期时间
            'iat': datetime.datetime.now(),  # 签发时间
            'data': {'username': self.username}  # 私有的声明
        }
        # jwt.encode(payload, 加盐, 加密方式, headers)
        token = jwt.encode(payload, settings.SECRET_KEY, 'HS256', headers)
        return token.decode('utf-8')

    @staticmethod
    def parse_payload(token, salt=None):
        """解密token"""
        if not salt: salt = settings.SECRET_KEY
        result = {"status":False,"data":None,"error":None}
        try:
            verified_payload = jwt.decode(token, salt, True)
            result["status"] = True
            result['data']=verified_payload
        except jwt.ExpiredSignatureError: result['error'] = 'token已失效'
        except jwt.DecodeError: result['error'] = 'token认证失败'
        except jwt.InvalidTokenError: result['error'] = '非法的token'
        return result


"""
 cross correlation test module
"""

import unittest
import redvox.common.cross_stats as cs
import numpy as np

SIGNAL_LENGTH = 100   # number of elements in signal
SAMPLE_RATE = 80.0    # sample rate in hz

class CrossCorrelationTests(unittest.TestCase):
    def setUp(self) -> None:
        self.sig = np.ndarray((SIGNAL_LENGTH,), float)
        self.sig_ref = np.ndarray((SIGNAL_LENGTH,), float)
        self.double_sig_ref = np.ndarray((SIGNAL_LENGTH * 2,), float)
        self.sample_rate = SAMPLE_RATE
        sign = 1.0
        for index in range(SIGNAL_LENGTH):
            self.sig[index] = (index % 25) * sign
            self.sig_ref[index] = ((index + 10) % 25) * sign
            self.double_sig_ref[index] = ((index - 7) % 25) * sign
            self.double_sig_ref[index + SIGNAL_LENGTH] = ((index - 10) % 25) * sign
            sign *= -1.0
        self.half_sig_ref = self.sig_ref[13:63]

    def test_xcorr_all(self):
        # sig length == ref length
        xcorr_indexes, xcorr, xcorr_offset_index, xcorr_offset_samples = cs.xcorr_all(self.sig, self.sig_ref)
        self.assertEqual(len(xcorr), 100)
        self.assertEqual(len(xcorr_indexes), 100)
        self.assertEqual(xcorr_offset_index, 40)
        self.assertEqual(xcorr_offset_samples, -10)
        # sig length < ref length
        xcorr_indexes, xcorr, xcorr_offset_index, xcorr_offset_samples = cs.xcorr_all(self.sig, self.double_sig_ref)
        self.assertEqual(len(xcorr), 299)
        self.assertEqual(len(xcorr_indexes), 299)
        self.assertEqual(xcorr_offset_index, 139)
        self.assertEqual(xcorr_offset_samples, 60)
        # sig length > ref length
        xcorr_indexes, xcorr, xcorr_offset_index, xcorr_offset_samples = cs.xcorr_all(self.sig, self.half_sig_ref)
        self.assertEqual(len(xcorr), 149)
        self.assertEqual(len(xcorr_indexes), 149)
        self.assertEqual(xcorr_offset_index, 76)
        self.assertEqual(xcorr_offset_samples, -23)

    def test_xcorr_main(self):
        xcorr_normalized_max, xcorr_offset_samples, xcorr_offset_seconds = \
            cs.xcorr_main(self.sig, self.sig_ref, self.sample_rate)
        self.assertAlmostEqual(xcorr_normalized_max, 0.9855, 4)
        self.assertEqual(xcorr_offset_samples, -10)
        self.assertEqual(xcorr_offset_seconds, -0.125)


import os

import pandas as pd

from env import get_db_url

'''
def new_titanic_data():
    url = get_db_url('titanic_db')
    return pd.read_sql('SELECT * FROM passengers', url)
'''

def get_titanic_data():
    '''
    returns a DataFrame from Titanic database
    '''

    filename = "titanic.csv"
    url = get_db_url('titanic_db')

    # if file is available locally, read it
    if os.path.isfile(filename):
        return pd.read_csv(filename)

    # if file not available locally, acquire data from SQL database
    # and write it as csv locally for future use
    else:
        # read the SQL query into a dataframe
        df =  pd.read_sql('SELECT * FROM passengers', url)

        # Write that dataframe to disk for later. Called "caching" the data for later.
        df.to_csv(filename, index_label = False)

        # Return the dataframe to the calling code
        return df

def get_iris_data():
    '''
    returns a DataFrame from iris_db
    with the species name and species id
    '''
    filename = 'iris.csv'
    sql = 'SELECT * FROM measurements JOIN species USING (species_id)'
    url = get_db_url('iris_db')

    if os.path.isfile(filename):
        return pd.read_csv(filename)
    else:
        df = pd.read_sql(sql, url)
        df.to_csv(filename, index_label = False)
        return df

def get_telco_data():
    '''
    returns everythin from telco_churn database
    '''
    filename = 'telco.csv'
    sql = '''
    SELECT * FROM customers
    JOIN contract_types USING (contract_type_id)
    JOIN internet_service_types USING (internet_service_type_id)
    JOIN payment_types USING (payment_type_id)
    '''
    url = get_db_url('telco_churn')

    if os.path.isfile(filename):
        return pd.read_csv(filename)
    else:
        df = pd.read_sql(sql, url)
        df.to_csv(filename, index_label = False)
        return df


import asyncio
import dsnet
from dsnet.core import QueryType
import pytest
from sqlalchemy import create_engine
import msgpack
from starlette.testclient import TestClient

from dsnetserver.main import app
from dsnetserver import __version__
from dsnetserver.main import DATABASE_URL, database
from dsnetserver.models import metadata, pigeonhole_message_table, broadcast_query_table
from dsnet.message import PigeonHoleNotification

db = None

def setup_module(module):
    global db
    db = create_engine(DATABASE_URL)
    metadata.create_all(db)

def teardown_module(module):
    asyncio.run(database.execute(pigeonhole_message_table.delete()))
    asyncio.run(database.execute(broadcast_query_table.delete()))

def test_root():
    client = TestClient(app)
    response = client.get("/")
    assert response.status_code == 200
    assert response.json() == {"message": "Datashare Network Server version %s" % __version__,
                               "server_version": __version__,
                               "core_version": dsnet.__version__,
                               "query_type": QueryType.CLEARTEXT}

def test_post_get_ph_message():
    client = TestClient(app)

    response = client.post("/ph/deadbeef", content=b'binary message')
    assert response.status_code == 200

    response = client.get("/ph/deadbeef")
    assert response.status_code == 200
    assert response.headers['Content-Type'] == 'application/octet-stream'
    assert response.content == b'binary message'

def test_post_get_ph_message_with_wrong_address():
    client = TestClient(app)

    response = client.get("/ph/unknown")
    assert response.status_code == 404

def test_get_ph_messages_by_shortened_address():
    client = TestClient(app)

    response = client.post("/ph/beefc0fe", content=b'binary message 1')
    assert response.status_code == 200

    response = client.post("/ph/beefc0c0", content=b'binary message 2')
    assert response.status_code == 200

    response = client.get("/ph/beefc0")
    assert response.status_code == 200
    assert response.headers['Content-Type'] == 'application/octet-stream'
    messages = msgpack.unpackb(response.content, raw=False)
    assert len(messages) == 2
    assert messages[0] == b'binary message 1'
    assert messages[1] == b'binary message 2'

@pytest.mark.timeout(5)
def test_post_broadcast():
    client = TestClient(app)

    with client.websocket_connect('/notifications') as websocket:
        response = client.post("/bb/broadcast", content=b'query payload')
        assert response.status_code == 200
        data = websocket.receive_bytes()
        assert data == b'query payload'

@pytest.mark.timeout(5)
def test_post_response():
    client = TestClient(app)

    with client.websocket_connect('/notifications') as websocket:
        response = client.post("/ph/deadbeef", content=b'response data')
        assert response.status_code == 200
        data = websocket.receive_bytes()
        assert data == PigeonHoleNotification('deadbe').to_bytes()


#!/usr/bin/env python3

import json
import subprocess
import sys

TARGET_TAG = 'latest'
ECS_CLUSTER = 'rust-ecs-prod'

def main():
    """CLI entrypoint of the program."""
    if len(sys.argv) < 2:
        usage()
    repository_name = sys.argv[1]
    images = get_images(repository_name)
    isRetry = None
    while True:
        selected_image_index = let_user_pick_image(images, isRetry)
        if selected_image_index == -1:
            exit(0)

        if selected_image_index is not None:
            break
        else:
            isRetry = True

    image = images[selected_image_index]
    imageTags = image.get("imageTags",[])
    if TARGET_TAG in imageTags:
        err(f"selected image already tagged as {TARGET_TAG}")

    eprint(f"selected option: {image}\n")
    manifest = get_image_manifest(repository_name, image["imageDigest"])
    retag_image(repository_name,manifest)
    image_pushed_at = format_time(image["imagePushedAt"])
    eprint(f"image pushed at {image_pushed_at} retaged as '{TARGET_TAG}'\n")
    if can_redeploy(repository_name):
        redeployed = force_redeploy()
        print("{}".format("successfully rollback and re-deploy" if redeployed else "Successfully rolled back the image, but redeploying the service with the same name failed"))
    else:
        eprint(f"Successfully rolled back the image, but no service with the same name to redeploy found")

def let_user_pick_image(images, isRetry=None):
    if isRetry is None:
        print("Please choosean image to rollback:\n")
        for idx, image in enumerate(images):
            image_pushed_at = format_time(image["imagePushedAt"])
            tags = ", ".join(image.get("imageTags", []))
            print("{}) {} {}".format(idx+1,image_pushed_at,"("+tags+")" if tags != "" else ""))
        print("")
    else:
        print(f"Invalid value, please check the list of possible values...\n")

    i = input("Enter image number (or 0 to exit): ")
    try:
        idx = int(i)
        if -1 < idx <= len(images):
            return idx-1
    except:
        pass
    return None

def get_images(repository_name):
    """Call ecr to get available images"""
    eprint("obtaining available images")
    try:
        out = json.loads( run_command([
            "aws", "ecr", "describe-images",
            "--repository-name", repository_name,
            "--no-paginate"
        ]).stdout)
    except subprocess.CalledProcessError as e:
        err(f"failed to get availabe images from repository: {e}" )

    return list(sorted(out["imageDetails"], key=lambda image: image["imagePushedAt"], reverse=True))

def get_image_manifest(repository_name, imageDigest):
    """Call ecr batch-get-image to get the image manifest"""
    try:
        out = json.loads( run_command([
            "aws", "ecr", "batch-get-image",
            "--repository-name", repository_name,
            "--image-ids", "imageDigest={}".format(imageDigest)
        ]).stdout)
    except subprocess.CalledProcessError as e:
        err(f"failed to get availabe images from repository: {e}" )

    return out["images"][0]["imageManifest"]

def retag_image(repository_name, manifest):
    """ """
    try:
        out = json.loads( run_command([
            "aws", "ecr", "put-image",
            "--repository-name", repository_name,
            "--image-manifest", manifest,
            "--image-tag", TARGET_TAG
        ]).stdout)
    except subprocess.CalledProcessError as e:
        err(f"failed to re-tag image as {TARGET_TAG}")

    return True

def can_redeploy(repository_name):
    """return True IFF there is a service with the same name of the repository."""
    try:
        out = json.loads( run_command([
            "aws", "ecs", "list-services",
            "--cluster", ECS_CLUSTER,
            "--no-paginate"
        ]).stdout)

        services = out["serviceArns"]
        for service in services:
            # last part of arn is the service name.
            if repository_name == service.split('/')[-1]:
                return True

        return False
    except subprocess.CalledProcessError as e:
        err(f"failed to list services in cluste {ECS_CLUSTER}")

def force_redeploy(service_name):
    """Force redeploy on ecs"""
    try:
        out = json.loads( run_command([
            "aws", "ecs", "update-service",
            "--cluster", ECS_CLUSTER,
            "--service", service_name,
            "--force-new-deployment"
        ]).stdout)

        return True
    except subprocess.CalledProcessError as e:
        err(f"failed to re-deploy service {service_name}")

###############
#  Utilities  #
###############

def format_time(time):
    return time.replace("T", " ").replace("+", " +")

def usage():
    """ print usage help and exit."""
    print("error: missing argument, you need to pass the repository name to use. e.g:")
    print("aws-rollback.py <repository name>")
    exit(1)

def eprint(*args, **kwargs):
    """Just like print(), but outputs on stderr."""
    print(*args, file=sys.stderr, **kwargs)

def err(*args, **kwargs):
    """Show the error message and exit with status 1."""
    eprint("error:", *args, **kwargs)
    exit(1)

def run_command(*args, **kwargs):
    """Run a CLI program capturing stdout. Raise an exception if it fails."""
    return subprocess.run(
        *args,
        stdout=subprocess.PIPE,
        check=True,
        **kwargs,
    )

####################
#  CLI Invocation  #
####################

if __name__ == "__main__":
    main()


from six import string_types
from pypif.obj.system.system import System
from pypif.obj.system.chemical.common.composition import Composition

class ChemicalSystem(System):
    """
    Representation of a general chemical system.
    """

    def __init__(self, uid=None, names=None, ids=None, classifications=None, source=None, quantity=None,
                 chemical_formula=None, composition=None, properties=None, preparation=None, sub_systems=None,
                 references=None, contacts=None, licenses=None, tags=None, **kwargs):
        """
        Constructor.

        :param uid: String with the permanent ID for this record.
        :param names: List of strings with common names of the chemical system.
        :param ids: List of dictionaries, strings, numbers, or :class:`.Id` objects that identify the system.
        :param classifications: List of dictionaries, strings, numbers, or :class:`.Classification` objects that
        classify the system.
        :param source: Dictionary, string, or :class:`.Source` object with the source of the system.
        :param quantity: Dictionary or :class:`.Quantity` object with the quantity of the system.
        :param chemical_formula: String with the chemical formula.
        :param composition: List of dictionaries or :class:`.Composition` objects that describe the composition of
                the chemical system.
        :param properties: List of dictionaries or :class:`.Property` objects with properties of the system.
        :param preparation: List of dictionaries or :class:`.ProcessStep` objects with the preparation
                information of the system.
        :param sub_systems: List of dictionaries or :class:`.System` objects with the subsystems of the system.
        :param references: List of dictionaries or :class:`.Reference` objects where information about the system
                is published.
        :param contacts: List of dictionaries, strings, or :class:`.Person` objects with people to contact for
                information about the system.
        :param licenses: List of dictionaries, strings, or :class:`.License` objects with licensing information
                for data about the system.
        :param tags: List of strings or numbers that are tags for this object.
        :param kwargs: Dictionary of fields that are not supported.
        """
        super(ChemicalSystem, self).__init__(uid=uid, names=names, ids=ids, classifications=classifications,
                                             source=source, quantity=quantity, properties=properties,
                                             preparation=preparation, sub_systems=sub_systems, references=references,
                                             contacts=contacts, licenses=licenses, tags=tags, **kwargs)
        self._chemical_formula = None
        self.chemical_formula = chemical_formula
        self._composition = None
        self.composition = composition
        self.category = kwargs['category'] if 'category' in kwargs else 'system.chemical'

    @property
    def chemical_formula(self):
        return self._chemical_formula

    @chemical_formula.setter
    def chemical_formula(self, chemical_formula):
        self._validate_type('chemical_formula', chemical_formula, string_types)
        self._chemical_formula = chemical_formula

    @chemical_formula.deleter
    def chemical_formula(self):
        self._chemical_formula = None

    @property
    def composition(self):
        return self._composition

    @composition.setter
    def composition(self, composition):
        self._validate_list_type('composition', composition, dict, Composition)
        self._composition = self._get_object(Composition, composition)

    @composition.deleter
    def composition(self):
        self._composition = None


# This is part of MiMiCPy

"""

This module contains Shell class to take care of low level
paramiko code for opening a remote shell and running commands

"""

from os.path import expanduser
import time
from ..utils.errors import MiMiCPyError, defaultHook

class Shell:
    def __init__(self, name, config=None):

        self.name = name

        if config == None:
            config = expanduser("~")+'/.ssh/config'

        self.config = config

        conf = self._lookup(name)

        if 'proxyjump' in conf:
            proxy_name, port = conf['proxyjump'].split(':')
            vmtransport = self._getssh(proxy_name).get_transport()
            dest_addr = (self._lookup(name)['hostname'], int(port))
            local_addr = (self._lookup(proxy_name)['hostname'], int(port))
            vmchannel = vmtransport.open_channel("direct-tcpip", dest_addr, local_addr)

            self.ssh = self._getssh(name, sock=vmchannel)

        else:
            self.ssh =  self._getssh(name)

        self.sftp = self.ssh.open_sftp()
        self.decoder = 'utf-8'
        self.is_open = True

        self.ssh_bg = None

    def _lookup(self, name):
        try:
            import paramiko
        except ImportError:
            raise MiMiCPyError("Paramiko python package not installed! Install it to remotely run commands")

        config = paramiko.SSHConfig()
        try:
            f = open(self.config)
        except FileNotFoundError:
            raise FileNotFoundError(f"No SSH config file found in {self.config}")

        config.parse(f)
        conf = config.lookup(name)

        return conf

    def _getssh(self, name, sock=None):
        try:
            import paramiko
        except ImportError:
            raise MiMiCPyError("Paramiko python package not installed! Install it to remote host")

        conf = self._lookup(name)

        if 'hostname' not in conf:
            raise MiMiCPyError(f"Hostname not found for {name} in SSH config file")
        if 'user' not in conf:
            raise MiMiCPyError(f"Username not found for {name} in SSH config file")

        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        if sock is None:
            ssh.connect(hostname=conf['hostname'], username=conf['user'], look_for_keys=False)
        else:
            ssh.connect(hostname=conf['hostname'], username=conf['user'], look_for_keys=False, sock=sock)

        return ssh

    def __del__(self):
        if not hasattr(self, 'is_open'): return
        if self.is_open:
            self.is_open = False
            self.sftp.close()
            self.ssh.close()
            if self.ssh_bg: self.ssh_bg.__del__()

    def runbg(self, cmd, hook=None, dirc='', query_rate=3):
        if self.ssh_bg == None:
            self.ssh_bg = _ShellBGRun(self.ssh, self.decoder)

        self.ssh_bg.run(f'cd {self.pwd()}/{dirc}')
        self.ssh_bg.run(self.loader_str)
        if not hook:
            hook = defaultHook

        out = self.ssh_bg.run(cmd.split(';')[-1], hook, query_rate)

        return out
        # do not destory ssh_bg, as this stops the process
        # kill it in the deconstructor

    def run(self, cmd, stdin=None, hook=None, fresh=False, dirc=''):
        if not fresh and self.loader_str:
            cmd = self.loader_str + ' ; ' + cmd

        cmd = f'cd {self.pwd()}/{dirc} ; ' + cmd

        tran = self.ssh.get_transport()
        chan = tran.open_session()
        chan.get_pty()
        stdout = chan.makefile('rb')
        chan.exec_command(cmd)

        if stdin:
            sin = chan.makefile('wb')
            sin.channel.send(stdin+'\n')
            sin.channel.shutdown_write()

        out = stdout.read().decode(self.decoder)

        if not fresh:
            out = out.replace(self.loader_out, '')

        if not hook:
            hook = defaultHook

        hook(cmd.split(';')[-1], out)

        return out

    def __enter__(self): return self

    def __exit__(self, exc_type, exc_val, exc_tb): self.__del__()

class _ShellBGRun:

    def __init__(self, ssh, decoder):
        self.chan = ssh.invoke_shell()
        self.stdout = ''
        self.decoder = decoder

    def queryStdout(self, buff = 1024):

       while self.chan.recv_ready():
           self.stdout += self.chan.recv(buff).decode(self.decoder)

    def run(self, cmd, hook=None, query_rate=0.2):

        self.queryStdout()

        startout = self.stdout

        self.chan.send(cmd+'\n')

        prev = startout

        while True:
            time.sleep(query_rate)

            self.queryStdout()

            if prev != self.stdout:
                text = self.stdout.replace(prev, '')

                prev = self.stdout

                if hook:
                    if hook(cmd, text): break

            else: break

        lines = self.stdout.replace(startout, '').splitlines()[1:-1]

        return '\n'.join(lines)

    def __del__(self):
        self.chan.close()


def ex97():
    fin=open('words.txt')
    flag=False
    word2=[]
    for line in fin:
        word=line.strip()
        #char=list(256)

        for i in range (0,len(word)-6):
            if word[i]==word[i+1] and word[i+2]==word[i+3] and word[i+4]==word[i+5]:
                word2.append(word)
                print(word)
                flag=True
    print (word2)
    return flag

def ex98():
    for i in range(100000,1000000):
        i1=i%10000
        if isPallindrome(i1):
            i2=(i+1)%100000
            if isPallindrome(i2):
                i3=i+2
                i31=(i3%100000)/10000
                if isPallindrome(i31):
                    i4=i+3
                    if isPallindrome(i4):
                        print(i)

def isPallindrome(num):
    n=num
    temp=0
    while (0<n):
        rem=n%10
        temp=(temp*10)+rem
        n=n/10
    print (temp)
    print (num)
    return (temp==num)


from appJar import gui
f= True

def press(button):
    #if button == "exit chat" app.
    if button =="enter app":

        app.prevFrame("Pages")
        user = app.getEntry("Username")
        app.label("hello-"+user)

def send(button):
    messa = app.getEntry("message")
    app.label("mess", messa)

def exit():
    f = False
    app.label("bye")

app = gui("FRAME STACK", "300x400", bg= "pink",font = 12)
app.startFrameStack("Pages")

app.startFrame() #second frame-chat frame

app.entry("message")
app.button("send",send)
app.button("exit",exit)

app.stopFrame()

app.startFrame() #first frame-username

app.addLabelEntry("Username")
app.setFont(10)
app.addButtons(["enter app","exit app"],press)

app.stopFrame()

app.stopFrameStack()

app.go()


# from tkinter import *
# from PIL import Image, ImageTk
from tkinter import Tk, Label, Button, Frame

def printing():
    # la = Label(var, text="Taimoor is best", fg='red', bg='black', borderwidth=5, relief="groove",
    #            anchor="e", width=20)
    # la.pack(side="left")
    # l = Frame(var, bg='black', borderwidth=5, relief="groove")
    # l.pack(side="top")
    # L1 = Label(l, text="taimoor is best", height=10)
    # L1.pack()
    l2.configure(text="HI")

var = Tk()
var.title("First GUI")
l2 = Label(var, text="Hello")
l2.pack()
var.geometry("600x400")
# var.maxsize(700, 800)
# var.minsize(700, 500)
v = Button(var, text="click the button", command=printing)
v.pack()
var.mainloop()

Taimoor Python/Code with harry/python GUI/GUI 1.py


import copy
import functools
import itertools
import logging
import operator
import time

import six

from . collections_ext import nameddict
from . itertools_ext import fxrange
from . rate_limits import truncated_exponential_backoff as teb

LOGGER = logging.getLogger(__name__)

class RetryDelaySeries(object):
    @classmethod
    def get(cls, name, **kwargs):
        return cls.definitions()[name](**kwargs)

    @classmethod
    def definitions(cls):
        return dict(
            truncated_exponential_backoff=cls.truncated_exponential_backoff,
            linear=cls.linear
        )

    @classmethod
    def truncated_exponential_backoff(cls, **kwargs):
        delay = kwargs.pop('delay')
        kwargs.setdefault('collision', 0)
        op = kwargs.get('op', None)
        if isinstance(op, six.string_types):
            kwargs.update(op=getattr(operator, op))
        while True:
            yield teb(delay, **kwargs)
            kwargs['collision'] = kwargs['collision'] + 1

    @classmethod
    def linear(cls, **kwargs):
        params = dict(
            start=kwargs['delay'],
            stop=kwargs.get('max_delay', 0),
            step=kwargs['step'],
        )
        return fxrange(**params)

class HA(object):
    HA_CONFIG_KEY = 'ha_retry'
    DEFAULT_HA_CONFIG = dict(
        default=dict(
            delay=30,
            max_retries=0,
            delay_policy='truncated_exponential_backoff',
            delay_config=dict(
                max_collisions=5
            )
        )
    )

    def __init__(self, ha_config=None, retry_delays_class=None):
        if ha_config is None:
            import docido_sdk.config as config
            ha_config = config.get(
                HA.HA_CONFIG_KEY,
                nameddict(HA.DEFAULT_HA_CONFIG)
            )
        self.ha_config = ha_config
        self._retry_delays_class = retry_delays_class or RetryDelaySeries

    @classmethod
    def catch(cls, catch_exception, config='default'):
        """Decorator class method catching exceptions raised by the wrapped
        member function. When exception is caught, the decorator waits
        for an amount of time specified in the `ha_config`.

        :param catch_exception: Exception class or tuple of exception classes.
        """
        def wrap(method):
            @functools.wraps(method)
            def wrapped_method(self, *args, **kwargs):
                assert isinstance(self, HA)
                delay_policy = self.ha_get_delay_policy(config)
                max_retries = self.ha_get_config(config).max_retries
                for retries in itertools.count():
                    try:
                        return method(self, *args, **kwargs)
                    except catch_exception as e:
                        res = self.ha_on_error(method, e, args, kwargs)
                        if res is not None:
                            args, kwargs = res
                        if max_retries and retries >= max_retries:
                            raise
                        tts = next(delay_policy)
                        time.sleep(tts)
            return wrapped_method
        return wrap

    def ha_get_config(self, name):
        return self.ha_config[name]

    def ha_get_delay_policy(self, config_name):
        """Build generator of delays to wait between each call

        :param string config_name: configuration name
        """
        config = self.ha_get_config(config_name)
        delay_policy_conf = config.delay_config
        delay_policy_conf = copy.deepcopy(delay_policy_conf)
        delay_policy_conf.update(delay=config.delay)
        return self._retry_delays_class.get(
            config.delay_policy,
            **delay_policy_conf
        )

    def ha_on_error(self, method, exc, my_args, my_kwargs):
        """Callback when exception is caught by the Ha.catch decorator.
        This method is meant to be overloaded.

        :param method: the method that failed.
        :param tuple args: arguments given to the method
        :param dict kwargs: keyword arguments given to the method

        :return: tuple (args, **kwargs) to override the arguments given
        to the next attempt, `None` otherwise.
        """
        pass


#!/usr/local/munkireport/munkireport-python3

import subprocess
import os
import plistlib
import sys
import platform
import re

def get_background_items():

    cmd = ['/usr/bin/sfltool', 'dumpbtm']
    proc = subprocess.Popen(cmd, shell=False, bufsize=-1, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    (output, unused_error) = proc.communicate()

    btm_info = output.decode("utf-8", errors="ignore")
    out = []
    uid = ""
    user = ""
    last_btm_index = 0

    for btm_items in btm_info.split('UUID: '):
        result = {'user': user}
        line_index = 0
        for item in btm_items.split('\n'):
            line_index += 1
            if " Records for UID " in item:
                uid = item.replace("Records for UID", "").split(':')[0].strip()
                cmd = ['/usr/bin/id', '-un', uid]
                proc = subprocess.Popen(cmd, shell=False, bufsize=-1, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                (output, unused_error) = proc.communicate()

                username = output.decode("utf-8", errors="ignore").strip()

                if username == "":
                    username = "System"

                result['user'] = username
                user = username

            elif " #" in item and ":" in item and item.split(":")[1] == "":
                last_btm_index = int(item.replace("#", "").replace(":", "").strip())
                result['btm_index'] = uid+" - "+str((last_btm_index - 1)) # We need to subtract 1 because of the UUID split
            elif "                 Name: " in item:
                name = item.replace("Name:", "").replace("(null)", "").strip()
                if name != "":
                    result['name'] = name

            elif "       Developer Name: " in item:
                developer_name = item.replace("Developer Name:", "").replace("(null)", "").strip()
                if developer_name != "":
                    result['developer_name'] = developer_name

            elif "                  URL: " in item:
                url = item.replace("URL:", "").replace("(null)", "").replace("file://", "").strip()
                if url != "":
                    result['url'] = url

            elif "      Team Identifier: " in item:
                result['team_id'] = item.replace("Team Identifier:", "").replace("(null)", "").strip()
            elif "    Assoc. Bundle IDs: " in item:
                result['assoc_bundle_id'] = item.replace("Assoc. Bundle IDs:", "").replace("[", "").replace("]", "").strip()
            elif "                 Type: " in item:
                result['type'] = item.replace("Type:", "").replace("(null)", "").split('(')[0].strip().title()
            elif "           Identifier: " in item:
                result['identifier'] = item.replace("Identifier:", "").strip()
            elif "           Generation: " in item:
                result['generation'] = item.replace("Generation:", "").strip()
            elif "    Parent Identifier: " in item:
                result['parent_id'] = item.replace("Parent Identifier:", "").strip()
            elif "      Executable Path: " in item:
                result['executable_path'] = item.replace("Executable Path:", "").strip()

            elif "          Disposition: " in item:
                disposition = item.replace("Disposition:", "").strip()
                if "disabled" in disposition:
                    result['state_enabled'] = 0
                elif "enabled" in disposition:
                    result['state_enabled'] = 1
                if "disallowed" in disposition:
                    result['state_allowed'] = 0
                elif "allowed" in disposition:
                    result['state_allowed'] = 1
                if "not visible" in disposition:
                    result['state_visible'] = 0
                elif "visible" in disposition:
                    result['state_visible'] = 1
                if "not notified" in disposition:
                    result['state_notified'] = 0
                elif "notified" in disposition:
                    result['state_notified'] = 1
            elif "  Embedded Item Identifiers:" in item:
                embedded_item_ids = ""
                line_index_process = line_index # We don't want to break the current index
                item = btm_items.split('\n')[line_index_process]

                while item != "":
                    embedded_item_ids = embedded_item_ids + item.split(': ')[1].strip() + "\n"
                    item = btm_items.split('\n')[line_index_process + 1]
                    line_index_process += 1

                result['embedded_item_ids'] = embedded_item_ids.strip()

        # Only add if we have actual data
        if len(result) > 4:

            # # Skip if dumpped item is com.apple.PackageKit.DeferredInstallFixup
            # if 'embedded_item_ids' in result and result['embedded_item_ids'] == "com.apple.PackageKit.DeferredInstallFixup":
            #     pass
            # elif 'name' not in result and 'developer_name' not in result and 'embedded_item_ids' not in result and 'identifier' in result and result['identifier'] == "Unknown Developer":
            # Skip if empty entry
            if 'name' not in result and 'developer_name' not in result and 'embedded_item_ids' not in result and 'identifier' in result and result['identifier'] == "Unknown Developer":
                pass
            else:

                if 'btm_index' not in result:
                    result['btm_index'] = uid+" - "+str(last_btm_index)

                out.append(result)

    return out

def getDarwinVersion():
    """Returns the Darwin version."""
    # Catalina -> 10.15.7 -> 19.6.0 -> 19
    darwin_version_tuple = platform.release().split('.')
    return int(darwin_version_tuple[0])

def main():
    """Main"""

    # Check if macOS 11 (Darwin 20) or higher
    if getDarwinVersion() >= 20:
        result = get_background_items()
    else:
        # Background items are not supported on macOS 10.15 and older
        result = []

    # Write background items results to cache
    cachedir = '%s/cache' % os.path.dirname(os.path.realpath(__file__))
    output_plist = os.path.join(cachedir, 'background_items.plist')
    try:
        plistlib.writePlist(result, output_plist)
    except:
        with open(output_plist, 'wb') as fp:
            plistlib.dump(result, fp, fmt=plistlib.FMT_XML)

if __name__ == "__main__":
    main()


import fingerprint_feature_extractor
import cv2
import numpy as np

# Read the fingerprint images
img1 = cv2.imread('enhanced/1.jpg', 0)
img2 = cv2.imread('enhanced/3.jpg', 0)

# Extract minutiae features from both fingerprints
FeaturesTerminations1, FeaturesBifurcations1 = fingerprint_feature_extractor.extract_minutiae_features(img1, spuriousMinutiaeThresh=10, invertImage=False, showResult=False, saveResult=False)
FeaturesTerminations2, FeaturesBifurcations2 = fingerprint_feature_extractor.extract_minutiae_features(img2, spuriousMinutiaeThresh=10, invertImage=False, showResult=False, saveResult=False)

# Combine both types of features for each fingerprint into a single list
Features1 = FeaturesTerminations1 + FeaturesBifurcations1
Features2 = FeaturesTerminations2 + FeaturesBifurcations2

# Create keypoint objects for each feature in both fingerprints
# kp1 = [cv2.KeyPoint(x=f.locX, y=f.locY, _size=10) for f in Features1]
# kp2 = [cv2.KeyPoint(x=f.locX, y=f.locY, _size=10) for f in Features2]
kp1 = [cv2.KeyPoint(x=f.locX, y=f.locY, size=10) for f in Features1]
kp2 = [cv2.KeyPoint(x=f.locX, y=f.locY, size=10) for f in Features2]

# Create a SIFT descriptor object
sift = cv2.SIFT_create()

# Calculate SIFT descriptors for each keypoint in both fingerprints
_, des1 = sift.compute(img1, kp1)
_, des2 = sift.compute(img2, kp2)

# Create a BFMatcher object to match SIFT descriptors
bf = cv2.BFMatcher()

# Match SIFT descriptors of both fingerprints using the BFMatcher object
matches = bf.match(des1, des2)

# Sort the matches in ascending order of their distances
matches = sorted(matches, key=lambda x: x.distance)

# Keep only the top 10% of matches based on distance
matches = matches[:int(len(matches) * 0.1)]

if len(matches) > 0:
    print("Fingerprints matched!")
else:
    print("Fingerprints do not match.")


import os
import re
dir = os.path.dirname(__file__)
os.chdir(dir)

def read_input_lines():
    with open('input.txt', 'r') as fh:
        return [x.strip() for x in fh.readlines()]

def read_input_text():
    with open('input.txt', 'r') as fh:
        return fh.read().strip()

lines = [x for x in read_input_lines()]

for i in lines:
    if len(i)%2 !=0:
        print(i,len(i))

item_list = []
for i in lines:
    a = i[0:len(i)//2]
    b = i[len(i)//2:]
    item = list(set(a)&set(b))[0]
    if item.lower() == item:
        prio = ord(item)-96
    else:
        prio = ord(item)-38
    item_list.append(prio)
print("part 1: ",sum(item_list))

item_list = []
while len(lines)>2:
    group = lines[0:3]
    lines = lines[3:]
    item = list(set(group[0]) & set(group[1]) & set(group[2]))[0]
    if item.lower() == item:
        prio = ord(item)-96
    else:
        prio = ord(item)-38
    item_list.append(prio)
print("part 2: ",sum(item_list))


# -*- coding: utf-8 -*-
# (c) 2021 Andreas Motl <andreas@hiveeyes.org>
# License: GNU Affero General Public License, Version 3
import dataclasses
import logging
from collections import OrderedDict
from typing import Dict, List, Optional
from urllib.parse import urljoin

from munch import Munch

logger = logging.getLogger(__name__)

@dataclasses.dataclass
class GrafanaDataModel:
    admin_stats: Optional[Dict] = dataclasses.field(default_factory=dict)
    dashboards: Optional[List[Munch]] = dataclasses.field(default_factory=list)
    dashboard_list: Optional[List[Munch]] = dataclasses.field(default_factory=list)
    datasources: Optional[List[Munch]] = dataclasses.field(default_factory=list)
    folders: Optional[List[Munch]] = dataclasses.field(default_factory=list)
    organizations: Optional[List[Munch]] = dataclasses.field(default_factory=list)
    users: Optional[List[Munch]] = dataclasses.field(default_factory=list)
    teams: Optional[List[Munch]] = dataclasses.field(default_factory=list)
    annotations: Optional[List[Munch]] = dataclasses.field(default_factory=list)
    snapshots: Optional[List[Munch]] = dataclasses.field(default_factory=list)
    notifications: Optional[List[Munch]] = dataclasses.field(default_factory=list)

@dataclasses.dataclass
class DashboardDetails:
    dashboard: Dict

    @property
    def panels(self) -> List:
        return self.dashboard.dashboard.get("panels", [])

    @property
    def annotations(self) -> List:
        return self.dashboard.dashboard.get("annotations", {}).get("list", [])

    @property
    def templating(self) -> List:
        return self.dashboard.dashboard.get("templating", {}).get("list", [])

@dataclasses.dataclass
class DashboardDataDetails:
    """
    Manage details concerned about "data"-relevant information,
    a subset of the complete bunch of dashboard information.
    """

    panels: List[Dict]
    annotations: List[Dict]
    templating: List[Dict]

    def to_munch(self):
        return Munch(panels=self.panels, annotations=self.annotations, templating=self.templating)

    @classmethod
    def from_dashboard_details(cls, dbdetails: DashboardDetails):
        ds_panels = cls.collect_data_nodes(dbdetails.panels)
        ds_annotations = cls.collect_data_nodes(dbdetails.annotations)
        ds_templating = cls.collect_data_nodes(dbdetails.templating)

        # Inline panel information into data details.
        targets = []
        for panel in ds_panels:
            panel_item = cls._format_panel_compact(panel)
            if "targets" in panel:
                for target in panel.targets:
                    target["_panel"] = panel_item
                    targets.append(target)

        return cls(panels=targets, annotations=ds_annotations, templating=ds_templating)

    @staticmethod
    def collect_data_nodes(element):
        """
        Select all element nodes which have a "datasource" attribute.
        """
        element = element or []
        items = []

        def add(item):
            if item is not None and item not in items:
                items.append(item)

        for node in element:
            if "datasource" in node and node["datasource"]:
                add(node)

        return items

    @staticmethod
    def _format_panel_compact(panel):
        """
        Return a compact representation of panel information.
        """
        attributes = ["id", "title", "type", "datasource"]
        data = OrderedDict()
        for attribute in attributes:
            data[attribute] = panel.get(attribute)
        return data

    @staticmethod
    def _format_data_node_compact(item: Dict) -> Dict:
        """
        Return a compact representation of an element concerned about data.
        """
        data = OrderedDict()
        data["datasource"] = item.get("datasource")
        data["type"] = item.get("type")
        for key, value in item.items():
            if "query" in key.lower():
                data[key] = value
        return data

    def queries_only(self):
        """
        Return a representation of data details information, only where query expressions are present.
        """
        # All attributes containing query-likes.
        attributes_query_likes = ["expr", "jql", "query", "rawSql", "target"]

        attributes = [
            # Carry over datasource and panel references for informational purposes.
            "datasource",
            "_panel",
        ] + attributes_query_likes

        def transform(section):
            new_items = []
            for item in section:
                new_item = OrderedDict()
                for key, value in item.items():
                    if key in attributes:
                        new_item[key] = value
                if any(attr in attributes_query_likes for attr in new_item.keys()):
                    # Filter annotations without any value.
                    if "target" in new_item and isinstance(new_item["target"], dict):
                        if new_item["target"].get("type") == "dashboard":
                            continue
                    # Unnest items with nested "query" slot.
                    for slot in ["query", "target"]:
                        if slot in new_item and isinstance(new_item[slot], dict) and "query" in new_item[slot]:
                            new_item["query"] = new_item[slot]["query"]
                    new_items.append(new_item)
            return new_items

        return DashboardDataDetails(
            panels=transform(self.panels),
            annotations=transform(self.annotations),
            templating=transform(self.templating),
        )

@dataclasses.dataclass
class DatasourceItem:
    uid: Optional[str] = None
    name: Optional[str] = None
    type: Optional[str] = None
    url: Optional[str] = None

    @classmethod
    def from_payload(cls, payload: any):
        if isinstance(payload, Munch):
            payload = dict(payload)
        if isinstance(payload, dict):
            return cls(**payload)
        if isinstance(payload, str):
            return cls(name=payload)
        raise TypeError(f"Unknown payload type for DatasourceItem: {type(payload)}")

@dataclasses.dataclass
class DatasourceExplorationItem:
    datasource: Munch
    used_in: List[Munch]
    grafana_url: str

    def format_compact(self):
        dsshort = OrderedDict(
            uid=self.datasource.get("uid"),
            name=self.datasource.name,
            type=self.datasource.type,
            url=self.datasource.url,
        )
        item = OrderedDict(datasource=dsshort)
        for dashboard in self.used_in:
            item.setdefault("dashboards", [])
            dbshort = OrderedDict(
                title=dashboard.dashboard.title,
                uid=dashboard.dashboard.uid,
                path=dashboard.meta.url,
                url=urljoin(self.grafana_url, dashboard.meta.url),
            )
            item["dashboards"].append(dbshort)
        return item

@dataclasses.dataclass
class DashboardExplorationItem:
    dashboard: Munch
    datasources: List[Munch]
    grafana_url: str

    def format(self, with_data_details: bool = False, queries_only: bool = False):
        """
        Generate a representation from selected information.

        - dashboard
        - datasources
        - details
            - panels/targets
            - annotations
            - templating
        """
        dbshort = OrderedDict(
            title=self.dashboard.dashboard.title,
            uid=self.dashboard.dashboard.uid,
            path=self.dashboard.meta.url,
            url=urljoin(self.grafana_url, self.dashboard.meta.url),
        )

        dsshort = []
        for datasource in self.datasources:
            item = OrderedDict(
                uid=datasource.get("uid"),
                name=datasource.name,
                type=datasource.type,
                url=datasource.url,
            )
            dsshort.append(item)

        data = Munch(dashboard=dbshort, datasources=dsshort)
        if with_data_details:
            details = self.collect_data_details(queries_only=queries_only)
            if not details.panels and not details.annotations and not details.templating:
                return None
            data.details = details.to_munch()
        return data

    def collect_data_details(self, queries_only: bool = False):
        """
        Collect details concerned about data from dashboard information.
        """

        dbdetails = DashboardDetails(dashboard=self.dashboard)
        dbdatadetails = DashboardDataDetails.from_dashboard_details(dbdetails)
        if queries_only:
            dbdatadetails = dbdatadetails.queries_only()
        return dbdatadetails


# -*- coding: utf-8 -*-

import os.path
import string

class Pinyin:
	data_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'pinyin.txt')

	def __init__(self):
		z = open(self.data_path)
		self.lines = z.readlines()
		z.close()

	def pinyin(self, str):
		r = ''
		for c in str:
			if c == u'-':
				r = r + c
				continue
			if c == u' ':
				r = r + c
				continue
			if c in string.letters:
				r = r + c
				continue
			if c in string.digits:
				r = r + c
				continue
			ret = self.find_it(c)
			if ret:
				r = r + ret + ','
		return r

	def find_it(self, x):
		ret = ''
		for line in self.lines:
			line = line.decode('utf-8')
			if line.find(x)!=-1:
				return line.split(":")[0].strip()


from setuptools import setup, find_packages

with open('README.md', 'r') as f:
    long_description = f.read()

setup(
    name='Flask-Discussion',
    version='0.1.1',

    description='Comment system integration for Flask applications',
    long_description=long_description,
    long_description_content_type='text/markdown',

    url='https://github.com/rmed/flask-discussion',

    author='Rafael Medina García',
    author_email='rafamedgar@gmail.com',

    license='MIT',

    classifiers=[
        'Development Status :: 4 - Beta',
        'Framework :: Flask',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: MIT License',
        'Programming Language :: Python'
    ],

    keywords='flask discussion comments commenting',

    packages=find_packages(),

    include_package_data=True,

    install_requires=[
        'Flask>=1.1.1',
    ]
)


from backtester.trading_system_parameters import TradingSystemParameters
from backtester.features.feature import Feature
from datetime import timedelta
from backtester.dataSource.csv_data_source import CsvDataSource
from backtester.timeRule.nse_time_rule import NSETimeRule
from backtester.executionSystem.simple_execution_system import SimpleExecutionSystem
from backtester.orderPlacer.backtesting_order_placer import BacktestingOrderPlacer
from backtester.trading_system import TradingSystem
from backtester.version import updateCheck
from backtester.constants import *
from backtester.features.feature import Feature
import pandas as pd
import numpy as np
from problem2_trading_params import  MyTradingParams

## Make your changes to the functions below.
## You need to specify features you want to use in getInstrumentFeatureConfigDicts() and getMarketFeatureConfigDicts()
## and create your predictions using these features in getPrediction()
## SPECIFY number of stocks in each basket below
## Don't change any other function
## The toolbox does the rest for you, from downloading and loading data to running backtest

class MyLeagueFunctions():

    def __init__(self):  #Put any global variables here
        self.count = 0
        self.params = {}
        self.lookback = 720  ## max number of historical datapoints you want at any given time

    def getPlayersInTeam(self):
        return 12

    ####################################
    ## FILL THESE FOUR FUNCTIONS BELOW ##
    ####################################

    '''
    Specify all Features you want to use by  by creating config dictionaries.
    Create one dictionary per feature and return them in an array.

    Feature config Dictionary have the following keys:

        featureId: a str for the type of feature you want to use
        featureKey: {optional} a str for the key you will use to call this feature
                    If not present, will just use featureId
        params: {optional} A dictionary with which contains other optional params if needed by the feature

    msDict = {'featureKey': 'ms_5',
              'featureId': 'moving_sum',
              'params': {'period': 5,
                         'featureName': 'basis'}}

    return [msDict]

    You can now use this feature by in getPRediction() calling it's featureKey, 'ms_5'
    '''

    def getPlayerFeatureConfigDicts(self):

        #############################################################################
        ### TODO: FILL THIS FUNCTION TO CREATE DESIRED FEATURES for each stock.   ###
        ### USE TEMPLATE BELOW AS EXAMPLE                                         ###
        #############################################################################
        mom1Dict = {'featureKey': 'mom_5',
                   'featureId': 'momentum',
                   'params': {'period': 5,
                              'featureName': 'F5'}}
        mom2Dict = {'featureKey': 'mom_10',
                   'featureId': 'momentum',
                   'params': {'period': 10,
                              'featureName': 'F5'}}
        return [mom1Dict, mom2Dict]

    def getTeamFeatureConfigDicts(self):
        #############################################################################
        ### TODO: FILL THIS FUNCTION TO CREATE a ratio feature                    ###
        ### AND DESIRED FEATURES for each ratio.                                  ###
        ### USE TEMPLATE BELOW AS EXAMPLE                                         ###
        #############################################################################

        # customFeatureDict = {'featureKey': 'custom_mrkt_feature',
        #                      'featureId': 'my_custom_mrkt_feature',
        #                      'params': {'param1': 'value1'}}
        return []

    '''
    Combine all the features to create the desired predictions for each stock.
    'predictions' is Pandas Series with stock as index and predictions as values
    We first call the holder for all the instrument features for all stocks as
        lookbackInstrumentFeatures = instrumentManager.getLookbackInstrumentFeatures()
    Then call the dataframe for a feature using its feature_key as
        ms5Data = lookbackInstrumentFeatures.getFeatureDf('ms_5')
    This returns a dataFrame for that feature for ALL stocks for all times upto lookback time
    Now you can call just the last data point for ALL stocks as
        ms5 = ms5Data.iloc[-1]
    You can call last datapoint for one stock 'ABC' as
        value_for_abs = ms5['ABC']

    Output of the prediction function is used by the toolbox to make further trading decisions and evaluate your score.
    '''

    def getPlayersAndPosition(self, time, updateNum, playerManager):

        # self.updateCount() - uncomment if you want a counter
        # holder for all the instrument features for all instruments
        #############################################################################################
        ###  TODO : FILL THIS FUNCTION TO RETURN A BUY (1) or SELL (0) prediction for each pair   ###
        ###  USE TEMPLATE BELOW AS EXAMPLE                                                        ###
        #############################################################################################

        lookbackPlayerFeatures = playerManager.getLookbackInstrumentFeatures()

        lookbackTeamFeatures = playerManager.getDataDf()
        mom1 = lookbackPlayerFeatures.getFeatureDf('mom_5')
        mom2 = lookbackPlayerFeatures.getFeatureDf('mom_10')

        #Create some factor to rank their skill on
        #You can create one or multiple factors
        #In this example we assume that this factor ranks players in order of forward to defense ability
        #Top ranked are best forwards, bottom ranked are best defenders
        factorValues = (mom1/mom2).iloc[-1]
        factorValues.sort_values(0, ascending=False, inplace=True)

        #Derive player ranks from factor values
        ranks = factorValues.rank(0)

        # Put players with no rank in the middle
        ranks.fillna(len(factorValues.index)/2, inplace=True)

        ## Choose players and their position

        players = pd.Series(index=ranks.index[(ranks<=self.getPlayersInTeam()/2)|(ranks>(len(ranks.index)-self.getPlayersInTeam()/2))])
        players[ranks<=self.getPlayersInTeam()/2] = 1
        players[ranks>=(len(ranks.index)-self.getPlayersInTeam()/2)] = 0

        return players

# def getPrediction(self, time, updateNum, instrumentManager, predictions):

#         # self.updateCount() - uncomment if you want a counter
#         # holder for all the instrument features for all instruments
#         lookbackInstrumentFeatures = instrumentManager.getLookbackInstrumentFeatures()

#         #############################################################################################
#         ###  TODO : FILL THIS FUNCTION TO RETURN A BUY (1) or SELL (0) prediction for each pair   ###
#         ###  USE TEMPLATE BELOW AS EXAMPLE                                                        ###
#         #############################################################################################

#         lookbackMarketFeatures = instrumentManager.getDataDf()
#         mom1 = lookbackInstrumentFeatures.getFeatureDf('mom_5')
#         mom2 = lookbackInstrumentFeatures.getFeatureDf('mom_10')
#         #Get latest factor values
#         factorValues = (mom1/mom2).iloc[-1]
#         factorValues.sort_values(0, inplace=True)

#         #Derive rank from factor values
#         ranks = factorValues.rank(0)

#         # Put stocks with no rank in the middle
#         ranks.fillna(len(predictions.index)/2, inplace=True)

#         if len(mom1.index) > 1:
#             oldFactorValues = (mom1/mom2).iloc[-2]
#             oldranks = oldFactorValues.rank(0)
#             oldranks.fillna(len(predictions.index)/2, inplace=True)
#         else:
#             oldranks = pd.Series(len(predictions.index)/2, index = ranks.index)

#         # Short position in the lowest ranked stocks
#         predictions[ranks<=self.getSymbolsInBasket()] = 0
#         # we are already short stocks that ranked lowest previous time, don't sell again them
#         predictions[(oldranks<=self.getSymbolsInBasket()) & (ranks<=self.getSymbolsInBasket())] = 0.25
#         predictions[ranks>self.getSymbolsInBasket()] = 0.5

#         # Long position in the highest ranked stocks
#         predictions[ranks>(len(predictions.index)-self.getSymbolsInBasket())] = 1
#         # we are already long stocks that ranked highest previous time, don't buy again them
#         predictions[(oldranks>(len(predictions.index)-self.getSymbolsInBasket())) &\
#                  (ranks>(len(predictions.index)-self.getSymbolsInBasket()))] = 0.75

#         return predictions

#     def updateCount(self):
#         self.count = self.count + 1

    ###########################################
    ##         DONOT CHANGE THESE            ##
    ###########################################

    def getLookbackSize(self):
        return self.lookback

    ###############################################
    ##  CHANGE ONLY IF YOU HAVE CUSTOM FEATURES  ##
    ###############################################

    def getCustomFeatures(self):
        return {'my_custom_feature_identifier': MyCustomFeatureClassName}

class MyCustomFeature(Feature):
    ''''
    Custom Feature to implement for instrument. This function would return the value of the feature you want to implement.
    1. create a new class MyCustomFeatureClassName for the feature and implement your logic in the function computeForInstrument() -

    2. modify function getCustomFeatures() to return a dictionary with Id for this class
        (follow formats like {'my_custom_feature_identifier': MyCustomFeatureClassName}.
        Make sure 'my_custom_feature_identifier' doesnt conflict with any of the pre defined feature Ids

        def getCustomFeatures(self):
            return {'my_custom_feature_identifier': MyCustomFeatureClassName}

    3. create a dict for this feature in getInstrumentFeatureConfigDicts() above. Dict format is:
            customFeatureDict = {'featureKey': 'my_custom_feature_key',
                                'featureId': 'my_custom_feature_identifier',
                                'params': {'param1': 'value1'}}
    You can now use this feature by calling it's featureKey, 'my_custom_feature_key' in getPrediction()
    '''
    @classmethod
    def computeForInstrument(cls, updateNum, time, featureParams, featureKey, instrumentManager):
        # Custom parameter which can be used as input to computation of this feature
        param1Value = featureParams['param1']

        # A holder for the all the instrument features
        lookbackInstrumentFeatures = instrumentManager.getLookbackInstrumentFeatures()

        # dataframe for a historical instrument feature (basis in this case). The index is the timestamps
        # atmost upto lookback data points. The columns of this dataframe are the stocks/instrumentIds.
        lookbackInstrumentValue = lookbackInstrumentFeatures.getFeatureDf('stockVWAP')

        # The last row of the previous dataframe gives the last calculated value for that feature (basis in this case)
        # This returns a series with stocks/instrumentIds as the index.
        currentValue = lookbackInstrumentValue.iloc[-1]

        if param1Value == 'value1':
            return currentValue * 0.1
        else:
            return currentValue * 0.5

if __name__ == "__main__":
    if updateCheck():
        print('Your version of the auquan toolbox package is old. Please update by running the following command:')
        print('pip install -U auquan_toolbox')
    else:
        tf = MyLeagueFunctions()
        tsParams = MyTradingParams(tf)
        tradingSystem = TradingSystem(tsParams)
    # Set onlyAnalyze to True to quickly generate csv files with all the features
    # Set onlyAnalyze to False to run a full backtest
    # Set makeInstrumentCsvs to False to not make instrument specific csvs in runLogs. This improves the performance BY A LOT
        tradingSystem.startTrading(onlyAnalyze=False, shouldPlot=True, makeInstrumentCsvs=True)


def toBool(value):
	if isinstance(value, str):
		try:
			s_as_int = int(value)
		except:
			s_as_int = None

		true_list = [ "true", "True", "yes", "Yes" ]
		false_list = [ "false", "False", "no", "No" ]

		if value in true_list or (s_as_int != None and s_as_int):
			return True
		elif value in false_list or (s_as_int != None and not s_as_int):
			return False
	elif isinstance(value, int):
		return value != 0
	elif isinstance(value, float):
		return value != 0.0
	else:
		return False


from flask import render_template
from department_app.forms.department_form import DepartmentForm
from department_app.views.departments_views_service import *

@app.route('/', methods=['GET'])
def main_page():
    """
    Renders 'main_page.html' template.
    Depending on the choice of the user redirects to 'departments' page or 'employees' page.

    :return: rendered 'main_page.html' template
    """
    app.logger.debug('main_page.html was rendered')
    return render_template('main_page.html')

@app.route('/departments', methods=['GET', 'POST'])
def departments():
    """
    Renders 'departments.html' template.
    Calls 'departments_service' method with additional logic.
    Redirects to 'create' page in case of creating a department.
    Redirects to 'department_get' page in case of view of a particular department.

    :return: rendered 'departments.html' template
    """
    form = DepartmentForm()
    filtered_departments = departments_service(form)
    app.logger.debug('departments.html was rendered')
    return render_template('departments.html', departments=filtered_departments, form=form)

@app.route('/departments/<int:department_id>', methods=['GET', 'POST'])
def department_get(department_id):
    """
    Renders 'department.html' template.
    Calls 'department_get_service' method with additional logic.
    Redirects to 'add_employee' page in case of adding new employee to department.
    Redirects to 'edit_department' page in case of editing department.

    :param department_id: id of department
    :return: rendered 'department.html' template
    """
    department, employees = department_get_service(department_id)
    app.logger.debug('department.html was rendered')
    return render_template('department.html', department=department,
                           employees=employees)

@app.route('/departments/<int:department_id>/add', methods=['GET', 'POST'])
def add_employee(department_id):
    """
    Renders 'employee_add.html' template.
    Calls 'add_employee_service' method with additional logic.

    :param department_id: id of department
    :return: rendered 'employee_add.html' template
    """
    department_info = add_employee_service(department_id)
    app.logger.debug('employee_add.html was rendered')
    return render_template('employee_add.html', department=department_info)

@app.route('/departments/<int:department_id>/edit', methods=['GET', 'POST'])
def edit_department(department_id):
    """
    Renders 'department_edit.html' template
    Calls 'edit_department_service' method with additional logic.
    Redirects to 'department_remove' when trying to remove department.

    :param department_id: id of department
    :return: rendered 'department_edit.html' template
    """
    department_info = edit_department_service(department_id)
    app.logger.debug('department_edit.html was rendered')
    return render_template('department_edit.html', department=department_info)

@app.route('/departments/create_department', methods=['GET', 'POST'])
def create():
    """
    Renders 'department_create.html' template
    Calls 'create_department_service' method with additional logic.

    :return: rendered 'department_create.html' template
    """
    create_department_service()
    app.logger.debug('department_create.html was rendered')
    return render_template('department_create.html')

@app.route('/departments/<int:department_id>/remove', methods=['GET', 'POST'])
def department_remove(department_id):
    """
    Renders 'department_remove.html' template.
    Calls 'department_name_id' method to get id and name of department.
    Calls 'department_remove_service' method with additional logic.
    Upon confirmation of the removal of the department, it is deleted from the database,
    redirects to 'removed_department' page.

    :param department_id: id of department
    :return: rendered 'department_remove.html' template
    """
    department_info = department_name_id(department_id)
    if request.method == 'POST':
        return department_remove_service(department_id)
    app.logger.debug(f'department_remove.html was rendered')
    return render_template('department_remove.html', department=department_info)

@app.route('/departments/removed_department', methods=['GET'])
def removed_department():
    """
    Renders 'removed_department.html' template.
    A flash notification about the success of the operation pops up.
    Arrow back redirects on 'departments' page.

    :return: rendered 'removed_department.html' template
    """
    app.logger.debug('Department was removed')
    flash('The department was successfully removed.', category='success')
    app.logger.debug('removed_department.html was rendered')
    return render_template('removed_department.html')


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2018/8/24 上午11:26
# @Author  : jlinka
# @File    : search.py

from pymongo import MongoClient
from bson.objectid import ObjectId
from app.main.complex_search import kmeans_search_condition
import jieba

# web = ['C114中国通信网', '中华人民共和国工业和信息化部', '中华人民共和国发展和改革委员会', '中华人民共和国科学技术部',
#        '中华人民共和国商务部', '深圳市发展和改革委员会', 'ITU', 'ITU中国', '3GPP', 'IMT-2020推进组', '通信世界网',
#        'OFweek光通讯网', '中国无线电管理', '电子发烧友', '电子产品世界', '新华网信息化', '新浪新闻', '新浪科技',
#        '新浪财经', '网易科技', '网易财经', '搜狐', '搜狐科技', '腾讯财经', '腾讯科技', '凤凰网', '凤凰科技', '21CN科技',
#        '环球科技', '飞象网', '中国科技网', '智通财经', 'CSDN', '和讯网', '华为', '中兴通讯', '上海诺基亚贝尔', '爱立信',
#        '大唐电信', '三星', '英特尔', '英特尔IQ', '高通Qualcomm', '中国移动设计院', '中国移动设计院', '中国联通研究院',
#        '中国电信北京研究院', '中国信通院']

class mongDbManger_web_showClass:
    __client = None
    __db = None
    __coll = None

    def __init__(self):
        print("__init__mongDbManger_webshow")

    def connect(self, url, port, user, password, database, table, auth):
        self.__client = MongoClient(url, port)
        self.__db = self.__client[auth]
        self.__db.authenticate(user, password)
        self.__db = self.__client[database]
        self.__coll = self.__db[table]

    def closeMongo(self):
        self.__client.close()

    def findOneWeek(self, dt):
        qureyCondition = {'createTime': {'$gt': dt}}
        cursor = self.__coll.find(qureyCondition)
        return cursor

    def find(self):
        cursor = self.__coll.find({"test": {'$exists': True}, 'isLimitSource': 1})
        return cursor

    def findSource(self):
        # cursor = self.__coll.find({"test": {'$exists': True}, 'isLimitSource': 1}, {"source": 1, "_id": 0})
        cursor = self.__coll.distinct('source', {"test": {'$exists': True}, 'isLimitSource': 1})
        return cursor

    def findAuther(self):
        # cursor = self.__coll.find({"test": {'$exists': True}, 'isLimitSource': 1}, {"auther": 1, "_id": 0})
        cursor = self.__coll.distinct('auther', {"test": {'$exists': True}, 'isLimitSource': 1, 'source': "微信公众号"})
        return cursor

    def findlike(self, filter, source, webs, auther):
        test_dict = {}
        source_list = []
        test_dict["test"] = {'$exists': True}
        test_dict['isLimitSource'] = 1
        if filter['content']['$regex'] != '':
            test_dict[list(filter.keys())[0]] = list(filter.values())[0]
        if source:
            for i in source:
                source_list.append(i)
                # test_dict['source'] = i
                if i == '网站':
                    source_list.extend(webs)
            test_dict['source'] = {'$in': source_list}
        else:
            if webs and auther:
                test_dict['$or'] = [{'source': {'$in': webs}},
                                    {'auther': {'$in': auther}}]
            if len(webs) != 0 and len(auther) == 0:
                test_dict['source'] = {'$in': webs}
            if len(webs) == 0 and len(auther) != 0:
                test_dict['auther'] = {'$in': auther}
            # test_dict['source'] = {'$in': webs}
            # test_dict['auther'] = {'$in': auther}

        cursor = self.__coll.find(test_dict)
        cursor.close()
        return cursor

    def findOne(self, filter):
        cursor = self.__coll.find_one({'_id': ObjectId(filter)})
        return cursor

    def test(self, test_dict):
        cursor = db.__coll.find(test_dict)
        for i in cursor:
            print(i['source'])
            print(i['auther'])

class PaginateLeave:
    def __init__(self, page, leave):
        self.total = leave.count()
        self.pages = int(self.total / 10)
        if self.total % 10 != 0:
            self.pages += 1
        if page == -1:
            self.page = self.pages
        else:
            self.page = page
        if self.page == 1:
            self.has_prev = False
        else:
            self.has_prev = True
        if self.page == self.pages:
            self.has_next = False
        else:
            self.has_next = True
        self.next_num = self.page + 1
        self.per_page = 10
        self.prev_num = self.page - 1
        self.current_num = self.total - (10 * (self.page - 1))
        if self.current_num > 10:
            self.current_num = 10
        self.items = []
        for i in range(self.current_num):
            yes_time = str(leave[self.prev_num * 10 + i].get('createTime'))
            yes_time = "".join(str(yes_time[0:4]) + "年" + str(yes_time[4:6]) + "月" + str(yes_time[6:8])) + "日"
            self.items.append(
                {'ObjectId': leave[self.prev_num * 10 + i].get('_id'),
                 'title': leave[self.prev_num * 10 + i].get('title'),
                 # 'createTime': leave[self.prev_num * 10 + i].get('createTime'),
                 'createTime': yes_time,
                 'content': leave[self.prev_num * 10 + i].get('content'),
                 'url': leave[self.prev_num * 10 + i].get('url'),
                 'source': leave[self.prev_num * 10 + i].get('source'),
                 'auther': leave[self.prev_num * 10 + i].get('auther')})

    def iter_pages(self, left_edge=2, left_current=2, right_current=5, right_edge=2):
        last = 0
        for num in range(1, self.pages + 1):
            if num <= left_edge or \
                    (self.page - left_current - 1 < num < self.page + right_current) \
                    or num > self.pages - right_edge:
                if last + 1 != num:
                    yield None
                yield num
                last = num

db = mongDbManger_web_showClass()
db.connect('server21.raisound.com', 24000, "webuser", "webuser1957", "web_show", "xinChuang_topic", "web_show")

def searchAll():
    cursor = db.find().sort('createTime', -1)
    # 关闭游标
    # cursor.close()
    # db.closeMongo()
    return cursor

def searchCondition(keyword, source, web, auther, order):
    filter = {}
    keyword = keyword
    condition = {}
    condition['$regex'] = keyword
    filter["content"] = condition
    cursor = db.findlike(filter, source, web, auther).sort('createTime', -1)
    if order == "智能排序":
        pass
    print(cursor.count())
    cursor.close()
    db.closeMongo()
    return cursor

def searchWeb():
    cursor = db.findSource()
    cursor.remove("微信公众号")
    db.closeMongo()
    return cursor

def searchOfficialAccounts():
    cursor = db.findAuther()
    db.closeMongo()
    return cursor

def findObjectId(keywords):
    cursor = db.findOne(keywords)
    db.closeMongo()
    return cursor

if __name__ == '__main__':
    # a = searchCondition("3GPP", "", "", "", "")
    a = searchAll()
    kmeans_search_condition(a)

    # a = db.findSource()
    # for i in a:
    #     print(i)


import pytest

from machaon.app import AppRoot
from machaon.process import ProcessMessage, Spirit, TempSpirit, Process
from machaon.ui.tk import tkLauncher

def gettext(log):
    return log.get(1.0, "end")

lastlineindex = ("end linestart -2 lines", "end linestart -2 lines lineend")
def getlastline(log):
    return log.get(*lastlineindex)
def getlasttag(log):
    return log.tag_names(lastlineindex[0])

@pytest.fixture
def approot():
    app = AppRoot()
    app.initialize(ui="tk", title="test", ignore_args=True)
    app.boot_ui()
    return app

def test_message_window(approot):
    spi = Spirit(approot)
    wnd = approot.get_ui()

    assert "\n" == gettext(wnd.log)
    wnd.insert_screen_text("message", "test-message")
    assert "test-message\n\n" == gettext(wnd.log)
    assert "test-message" == getlastline(wnd.log)

    wnd.insert_screen_text("error", "test-error-message")
    assert "test-error-message" == getlastline(wnd.log)
    assert ("error",) == getlasttag(wnd.log)

    wnd.insert_screen_text("hyperlink", "test-hyperlink", link="www.hellowork.go.jp", linktag="message-em")
    assert "test-hyperlink" == getlastline(wnd.log)
    assert set(("message-em","clickable","hlink-1")) == set(getlasttag(wnd.log))

    #wnd.log.mark_set("CURRENT", "end")
    l1 = wnd.log.get("end linestart -2 lines", "end linestart -1 lines")
    l2 = wnd.log.get("end linestart -3 lines", "end linestart -2 lines")

    wnd.message_handler(ProcessMessage(tag="delete-message", line=-1, count=1))
    assert "test-error-message" == getlastline(wnd.log)

    #wnd.run_mainloop()

def test_textindex(approot):
    from machaon.ui.tk import TextIndex as textindex

    assert textindex("1.2").line == 1 and textindex("1.2").char == 2
    assert str(textindex("2.3")) == "2.3"
    assert str(textindex("10.2").move(char=5)) == "10.5"
    assert str(textindex("10.2").move(line=0, char=0)) == "0.0"
    assert str(textindex(line=3, char="end")) == "3.end"
    i = textindex("9.7")
    assert i.shifted(line=-1, char=1).string() == "8.8"
    assert i.string() == "9.7"
    assert textindex("1.0").compare(textindex("1.3")) == 1
    assert textindex("1.2").compare(textindex("2.3")) == 1
    assert textindex("2.3").compare(textindex("2.3")) == 0
    assert textindex("2.3").compare(textindex("1.2")) == -1


class Solution:
    def find_dist(self, x, y):
        return math.sqrt(x**2 + y**2)

    def kClosest(self, points: List[List[int]], k: int) -> List[List[int]]:
        adict =  []

        for x, y in points:
            dist = self.find_dist(x, y)
            adict.append([(x, y), dist])
        adict.sort(key= lambda x: x[1])
        return [p for p, v in adict[:k]]

class Solution:
    def get_distance(self, x, y):
        return x*x + y*y

    def kClosest(self, P, k):
        heap = []
        for i, (x, y) in enumerate(P):
            d = self.get_distance(x, y)
            if len(heap) == k:
                heappushpop(heap, (-d, i))
            else:
                heappush(heap, (-d, i))
        return [P[i] for (_, i) in heap]


import math
import torch
import torch.nn as nn
from torch.nn.init import xavier_normal_, normal_
import torch.nn.functional as F

class GraphConvolution(nn.Module):
    # @article{kipf2016semi,
    #   title={Semi-Supervised Classification with Graph Convolutional Networks},
    #   author={Kipf, Thomas N and Welling, Max},
    #   journal={arXiv preprint arXiv:1609.02907},
    #   year={2016}
    # }
    """
    Reference code:
        https://github.com/tkipf/pygcn/blob/master/pygcn/layers.py
    Simple GCN layer, similar to https://arxiv.org/abs/1609.02907
    """

    def __init__(self, in_features, out_features, bias=True):
        super(GraphConvolution, self).__init__()
        self.in_features = in_features
        self.out_features = out_features
        self.weight = nn.Parameter(torch.FloatTensor(in_features, out_features))
        if bias:
            self.bias = nn.Parameter(torch.FloatTensor(out_features))
        else:
            self.register_parameter('bias', None)
        self.reset_parameters()

    def reset_parameters(self):
        stdv = 1. / math.sqrt(self.weight.size(1))
        self.weight.data.uniform_(-stdv, stdv)
        if self.bias is not None:
            self.bias.data.uniform_(-stdv, stdv)

    def forward(self, input, adj):
        support = torch.mm(input, self.weight)
        output = torch.spmm(adj, support)
        if self.bias is not None:
            return output + self.bias
        else:
            return output

    def __repr__(self):
        return self.__class__.__name__ + ' (' \
               + str(self.in_features) + ' -> ' \
               + str(self.out_features) + ')'

class FactorizationMachine(nn.Module):
    def __init__(self, n_users, n_items, u_input_features, i_input_features, embedding_size):
        super(FactorizationMachine, self).__init__()
        # define first fm embedding table and parameters
        self.first_user_embedding = nn.Embedding(n_users, 1)
        self.first_item_embedding = nn.Embedding(n_items, 1)
        self.user_W = nn.Parameter(torch.empty(size=(u_input_features, 1)))
        self.item_W = nn.Parameter(torch.empty(size=(i_input_features, 1)))

        # define second fm embedding table and parameters
        self.second_user_embedding = nn.Embedding(n_users, embedding_size)
        self.second_item_embedding = nn.Embedding(n_items, embedding_size)
        self.user_V = nn.Parameter(torch.empty(size=(u_input_features, embedding_size)))
        self.item_V = nn.Parameter(torch.empty(size=(i_input_features, embedding_size)))

        # define bias
        self.bias = nn.Parameter(torch.zeros(1, ))

        # parameters initialization
        self._init_weights()

    def _init_weights(self):
        xavier_normal_(self.first_user_embedding.weight.data)
        xavier_normal_(self.first_item_embedding.weight.data)
        xavier_normal_(self.user_W.data)
        xavier_normal_(self.item_W.data)

        xavier_normal_(self.second_user_embedding.weight.data)
        xavier_normal_(self.second_item_embedding.weight.data)
        xavier_normal_(self.user_V.data)
        xavier_normal_(self.item_V.data)

    def first_linear_layer(self, first_embeddings):
        """
        Args:
            first_embeddings: batch_size * (filed_num * embedding_size)
        """
        return torch.sum(first_embeddings, dim=1)  # batch_size

    def second_layer(self, second_embeddings):
        """
        Args:
            second_embeddings: batch_size * filed_num * embeding_size
        """
        square_of_sum = torch.sum(second_embeddings, dim=1) ** 2  # batch_size * embedding_size
        sum_of_square = torch.sum(second_embeddings ** 2, dim=1)  # batch_size * embedding_size
        output = square_of_sum - sum_of_square
        output = torch.sum(output, dim=1)  # batch_size
        output = 0.5 * output
        return output

    def forward(self, first_embeddings, second_embeddings):
        return self.bias + self.first_linear_layer(first_embeddings) + self.second_layer(second_embeddings)

    def get_first_embedding(self, user, item, user_feat_mat, item_feat_mat):
        user_feat_embedding = torch.mm(user_feat_mat, self.user_W)  # batch_sizer * 1
        item_feat_embedding = torch.mm(item_feat_mat, self.item_W)  # batch_sizer * 1

        first_embedding = []
        first_embedding.append(self.first_user_embedding(user))
        first_embedding.append(self.first_item_embedding(item))
        first_embedding.append(user_feat_embedding[user])
        first_embedding.append(item_feat_embedding[item])
        return torch.cat(first_embedding, dim=1)  # batch_size * 4

    def get_second_embedding(self, user, item, user_feat_mat, item_feat_mat):
        user_feat_embedding = torch.mm(user_feat_mat, self.user_V)  # batch_sizer * emb_size
        item_feat_embedding = torch.mm(item_feat_mat, self.item_V)  # batch_sizer * emb_size
        second_embedding = torch.stack([self.second_user_embedding(user),
                                       self.second_item_embedding(item),
                                       user_feat_embedding[user],
                                       item_feat_embedding[item]], dim=1)
        return second_embedding  # batch_size * 4 * emb_size

class MLPLayers(nn.Module):
    """
    Reference code:
        https://github.com/RUCAIBox/RecBole/blob/master/recbole/model/layers.py
    """
    def __init__(self, layers, dropout, use_bn=False):
        super(MLPLayers, self).__init__()
        self.layers = layers
        self.dropout = dropout
        self.use_bn = use_bn

        mlp_modules = []
        for idx, (input_size, output_size) in enumerate(zip(self.layers[:-1], self.layers[1:])):
            mlp_modules.append(nn.Dropout(p=self.dropout))
            mlp_modules.append(nn.Linear(input_size, output_size))
            if self.use_bn:
                mlp_modules.append(nn.BatchNorm1d(num_features=output_size))
            mlp_modules.append(nn.ReLU())

        self.mlp_layers = nn.Sequential(*mlp_modules)
        self.apply(self.init_weights)

    def init_weights(self, module):
        # We just initialize the module with normal distribution as the paper said
        if isinstance(module, nn.Linear):
            normal_(module.weight.data, 0, 0.01)

    def forward(self, input_feature):
        return self.mlp_layers(input_feature)

class LightGCNLayer(nn.Module):
    """
    Reference code:
        https://github.com/RUCAIBox/RecBole/blob/master/recbole/model/general_recommender/lightgcn.py
    """
    def __init__(self, n_users, n_items, embedding_size):
        super(LightGCNLayer, self).__init__()
        # load parameters
        self.n_users = n_users
        self.n_items = n_items
        self.embedding_size = embedding_size

        # define layer
        self.user_embedding = nn.Embedding(self.n_users, self.embedding_size)
        self.item_embedding = nn.Embedding(self.n_items, self.embedding_size)

        self.apply(self.init_weights)

    def init_weights(self, m):
        if isinstance(m, nn.Embedding):
            xavier_normal_(m.weight.data)

    def forward(self, n_layers, adj):
        all_embeddings = torch.cat([self.user_embedding.weight, self.item_embedding.weight], dim=0)
        embeddings_list = [all_embeddings]
        for layer_idx in range(n_layers):
            all_embeddings = torch.sparse.mm(adj, all_embeddings)
            embeddings_list.append(all_embeddings)
        lightgcn_all_embeddings = torch.stack(embeddings_list, dim=1)
        lightgcn_all_embeddings = torch.mean(lightgcn_all_embeddings, dim=1)

        user_lightgcn_embeddings, item_lightgcn_embeddings = torch.split(lightgcn_all_embeddings,
                                                               [self.n_users, self.n_items])
        return user_lightgcn_embeddings, item_lightgcn_embeddings

class StackGCN(nn.Module):
    def __init__(self, input_shape, output_shape, n_layers):
        super(StackGCN, self).__init__()
        self.n_layers = n_layers
        hidden_shape = (input_shape + output_shape) // 2
        self.layers = [input_shape] + [hidden_shape] * (self.n_layers - 1) + [output_shape]

        self.GCNLayer = nn.Sequential()
        for i, (input_feat, output_feat) in enumerate(zip(self.layers[:-1], self.layers[1:])):
            self.GCNLayer.add_module(f'GCN{i + 1}', GraphConvolution(input_feat, output_feat))

    def forward(self, input, adj, dropout):
        for i in range(self.n_layers - 1):
            input = self.GCNLayer[i](input, adj)
            input = F.relu(input)
            input = F.dropout(input, p=dropout, training=self.training)
        output = self.GCNLayer[i + 1](input, adj)
        return output


import sys, os, traceback
import ast
from statistics import mean

import torch
from sklearn.metrics import mean_absolute_error,mean_squared_error
import numpy as np
from sklearn.decomposition import PCA
import json
# from sklearn.manifold import TSNE
from torchmetrics import MeanAbsolutePercentageError
def csm(A,B):
    num=np.dot(A,B.T)/(np.sqrt(np.sum(A**2,axis=1)[:,np.newaxis])*np.sqrt(np.sum(B**2,axis=1))[np.newaxis,:])
    return num
def adjustScoreForMatrix(scores):
    try:
        minValue=np.min(scores)
        maxValue=np.max(scores)
        distance=maxValue-minValue
        scores=(scores-minValue)/distance
    except Exception as e:
        traceback.print_exc()
    return scores

def getReductionEmb(nl_vecs,code_vecs,reductType,reductSize):
    lenNLVecs=len(nl_vecs)
    lenCodeVecs=len(code_vecs)
    nl_vecs_transform=[]
    code_vecs_transform=[]
    all_vecs=nl_vecs+code_vecs
    # print('len all {}'.format(len(all_vecs)))
    if reductType=='adhoc':
        all_vecs_transform=[element[:reductSize] for element in all_vecs]
        nl_vecs_transform = all_vecs_transform[:lenNLVecs]
        code_vecs_transform = all_vecs_transform[lenNLVecs:]
    elif reductType=='pca':
        pca = PCA(n_components=reductSize)
        all_vecs_transform =pca.fit_transform(all_vecs)
        nl_vecs_transform = all_vecs_transform[:lenNLVecs]
        code_vecs_transform = all_vecs_transform[lenNLVecs:]
    # else:
    #     tsne = TSNE(n_components=reductSize,
    #                 perplexity=40,
    #                 random_state=42,
    #                 n_iter=5000,
    #                 n_jobs=-1)
    #     all_vecs_transform = tsne.fit_transform(all_vecs)
    #     nl_vecs_transform = all_vecs_transform[:lenNLVecs]
    #     code_vecs_transform = all_vecs_transform[lenNLVecs:]

    return nl_vecs_transform.tolist(),code_vecs_transform.tolist()

def createDirIfNotExist(fopOutput):
    try:
        # Create target Directory
        os.makedirs(fopOutput, exist_ok=True)
        #print("Directory ", fopOutput, " Created ")
    except FileExistsError:
        print("Directory ", fopOutput, " already exists")