# -*- coding: utf-8 -*-
"""
Created on Thu Aug 22 20:33:30 2019

@author: user
"""

import sys
import numpy as np
from keras.datasets import mnist

(x_train,y_train),(x_test,y_test)=mnist.load_data()
images,labels=(x_train[0:1000].reshape(1000,28*28)/255,y_train[0:1000])

one_hot_labels=np.zeros((len(labels),10))
# i= row no. ;  l= value
for i,l in enumerate(labels):
    one_hot_labels[i][l]=1
labels=one_hot_labels
# It gives 1 value to the row and column where the actual value is in the label
print(labels[256:384])
test_images=x_test.reshape(len(x_test),28*28)/255
test_labels=np.zeros((len(y_test),10))
for i,l in enumerate(y_test):
    test_labels[i][l]=1
print(test_labels.shape) # (10000, 10)
print(test_images.shape) # (10000, 784)
print(range(len(test_images)))

np.random.seed(1)
num_label=10
pixels_per_image=784
alpha,iterations=(2,300)
input_rows=28
input_cols=28
kernel_rows=3
kernel_cols=3
num_kernels=16
batch_size=128
hidden_size=((input_rows-kernel_rows)*(input_cols-kernel_cols)*num_kernels)
w1=0.2*np.random.random((hidden_size,num_label))-0.1
print(w1.shape) # (10000, 10)
kernels=0.02*np.random.random((kernel_rows*kernel_cols,num_kernels))-0.01
def tanh(x):
    return np.tanh(x)

def tanh2deriv(x):
    return (1-(x**2))

def softmax(x):
    temp=np.exp(x)
    return (temp/np.sum(temp,axis=1,keepdims=True))

def get_image_section(layer,row_from,row_to,col_from,col_to):
    section=layer[:,row_from:row_to,col_from:col_to]
    return section.reshape(-1,1,row_to-row_from,col_to-col_from)
i=0
for j in range(iterations):
    cnt=0
    for i in range(int(len(images)/batch_size)):
        batch_start,batch_end=((i*batch_size),((i+1)*batch_size))
        l0=images[batch_start:batch_end]
        l0=l0.reshape(l0.shape[0],28,28)
       # print(l0.shape) # (128, 28, 28)
        sects=list()
        # The following double loop run for 25*25=625 times
        for row_start in range(l0.shape[1]-kernel_rows):
            for col_start in range(l0.shape[2]-kernel_cols):
                sect=get_image_section(l0,row_start,row_start+kernel_rows,col_start,col_start+kernel_cols)
               # print(sect.shape) # (128, 1, 3, 3)
                sects.append(sect)
       # print(sects[1].shape) # (128, 1, 3, 3)
        extended_input=np.concatenate(sects,axis=1)
        es=extended_input.shape
      #  print(es) # (128, 625, 3, 3)
        flattened_input=extended_input.reshape(es[0]*es[1],-1)
      #  print(flattened_input.shape) # (80000, 9)
        kernel_output=np.dot(flattened_input,kernels)
        #print(kernel_output.shape) # (80000, 16)
        l1=tanh(kernel_output.reshape(es[0],-1))
       # print(kernel_output.reshape(es[0],-1).shape) # (128, 10000)
        dropout=np.random.randint(2,size=l1.shape)
        l1*=dropout*2
        l2=softmax(np.dot(l1,w1)) # (128, 10)
        for k in range(batch_size):
            labelset=labels[batch_start+k:batch_start+k+1]
            inc=int(np.argmax(l2[k:k+1])==np.argmax(labelset))
            cnt+=inc
        l2_delta=(labels[batch_start:batch_end]-l2)/(batch_size*l2.shape[0])
        l1_delta=np.dot(l2_delta,w1.T)*tanh2deriv(l1)
        l1_delta*=dropout
        w1+=alpha*np.dot(l1.T,l2_delta)
        l1_delta_reshape=l1_delta.reshape(kernel_output.shape)
       # print(l1_delta_reshape.shape) # (80000, 16)
        k_update=flattened_input.T.dot(l1_delta_reshape)
       # print(k_update.shape) # (9, 16)
        kernels-=alpha*k_update
    test_cnt=0
    for i in range(len(test_images)):
        l0=test_images[i:i+1]
        l0=l0.reshape(l0.shape[0],28,28)
        sects=list()
        for row_start in range(l0.shape[1]-kernel_rows):
            for col_start in range(l0.shape[2]-kernel_cols):
                sect=get_image_section(l0,row_start,row_start+kernel_rows,col_start,col_start+kernel_cols)
                sects.append(sect)
        t_extended_input=np.concatenate(sects,axis=1)
      #  print(t_extended_input.shape)
        es=t_extended_input.shape
      #  print(es[0],es[1])
        t_flattened_input=t_extended_input.reshape(es[0]*es[1],-1)
       # print(t_flattened_input.shape)
        kernel_output=np.dot(t_flattened_input,kernels)
        l1=tanh(kernel_output.reshape(es[0],-1))
        l2=softmax(np.dot(l1,w1))
        test_cnt+=int(np.argmax(l2)==np.argmax(test_labels[i:i+1]))
    sys.stdout.write('\r'+ 'I:'+ str(j) + ' Test_acc: '+ str(test_cnt/float(len(test_images))) + ' Train_acc: '+ str(cnt/float(len(images))))


from aocd import data

def walk(pos, dir):
    x, y = pos
    return {
        'U': (x + 1, y),
        'D': (x - 1, y),
        'L': (x, y - 1),
        'R': (x, y + 1)
    }[dir]

def distance(pos):
    return abs(pos[0]) + abs(pos[1])

def trace(steps):
    path, pos = set(), (0, 0)
    step_count = dict()
    nsteps = 0
    for step in steps.split(','):
        for _ in range(int(step[1:])):
            nsteps += 1
            pos = walk(pos, step[0])
            if pos not in step_count:
                step_count[pos] = nsteps
            path.add(pos)
    return path, step_count

def part1(steps1, steps2):
    crossings = trace(steps1)[0].intersection(trace(steps2)[0])
    closest = 1000000000000000
    for crossing in crossings:
        if distance(crossing) < closest:
            closest = distance(crossing)
    return closest

def part2(steps1, steps2):
    path1, step_count1 = trace(steps1)
    path2, step_count2 = trace(steps2)
    crossings = path1.intersection(path2)
    closest = 100000000000
    for crossing in crossings:
        dist = step_count1[crossing] + step_count2[crossing]
        if dist < closest:
            closest = dist
    return closest

steps1, steps2 = data.splitlines()

print('part 1:', part1(steps1, steps2))
print('part 2:', part2(steps1, steps2))


# Минимум из 3-х чисел

def my_min_3(a, b, c):
    if a < b:
        if a < c:
            return a
        else:
            return c
    else:
        if b < c:
            return b
        else:
            return c

a = int(input())
b = int(input())
c = int(input())
print(my_min_3(a, b, c))


import time
import uuid
from utilities.connections import create_connection, set_handlers, subscribe_all, send_all
from utilities.tempfiles import print_scan_file_status
from pprint import pprint
import os
import tempfile

conn_dict = {'acquire-request-topic': "/topic/org.eclipse.scanning.request.acquire.topic",
             'acquire-response-topic': "/topic/org.eclipse.scanning.response.acquire.topic",
             }

# Used to maintain reference to the same request.
unique_id = str(uuid.uuid4())
# Tempfile to use for scan. Can check file size to determine if used.
fd, tempfile = tempfile.mkstemp(".nxs", prefix="acquire_test")

os.close(fd)

print_scan_file_status(tempfile)

class StatusTopicHandler(object):

    def on_message(self, headers, data):

        pprint(data)

        if (data['status'] == "COMPLETE"):
            print("Complete.")
            print_scan_file_status(tempfile)

        if (data['status'] == "FAILED"):
            print("Failed.")
            print_scan_file_status(tempfile)

handlers = {'acquire-response-topic': StatusTopicHandler(),
            }

conn = create_connection()

set_handlers(conn, handlers)

subscribe_all(conn, conn_dict, handlers)

submission_time = int(time.time() * 1000)

acquire_request = {"@type":"AcquireRequest",
                   "uniqueId":"c8f12aee-d56a-49f6-bc03-9c7de9415674",
                   "detectorName":"mandelbrot",
                    "detectorModel":{"@type":"MandelbrotModel",
                                     "name":"mandelbrot",
                                     "exposureTime":0.01,
                                     "maxIterations":500,
                                     "escapeRadius":10.0,
                                     "columns":301,
                                     "rows":241,
                                     "points":1000,
                                     "maxRealCoordinate":1.5,
                                     "maxImaginaryCoordinate":1.2,
                                     "realAxisName":"xNex",
                                     "imaginaryAxisName":"yNex",
                                     "enableNoise":"false",
                                     "noiseFreeExposureTime":5.0,
                                     "timeout":-1
                                     },
                   "filePath":tempfile,
                   "status":"NONE"
                   }

send_all(conn, conn_dict,[("acquire-request-topic", acquire_request, 0)])

# Required to keep Python running indefinitely.
raw_input()

# You can mess around with the file, then press enter in the terminal to delete it.
os.remove(tempfile)


# Creacion variable "Dias de Fraude"
df["dias_fraude"] = df.FECHA_DETECCION - df.FECHA_INICIAL
df["dias_fraude"] = df.dias_fraude.dt.days

# Verificacion de que no existan dias negativos
df[df["FECHA_DETECCION"] < df["FECHA_INICIAL"]]

# Verificacion de cantidad de filas duplicadas
duplicate_rows_df = df[df.duplicated()]
print("number of duplicate rows: ", duplicate_rows_df.shape)

# Verificacion datos faltantes
print(df.isnull().sum())


## imports
import pandas as pd

## fonctions

# Uniformisation des dates au format Datetime
# prend en argument le dataframe et la colonne (string) contenant la date
def date_datetime_rename(df,col):
    df.rename(columns={col: "date"}, inplace=True) #renomme la colonne en "date"
    df[col] = pd.to_datetime(df[col])

# Renomme les colonnes du titre de l'article et du journal
# prend en argument le dataframe et la colonne (string) contenant le titre de l'article
def title_rename(df,col_title, col_journal):
    df.rename(columns={col_title: "title"}, inplace=True) #renomme la colonne en "title"
    df.rename(columns={col_journal: "journal"}, inplace=True) #renomme la colonne en "journal"

# Met chaîne de caractère en minuscule : afin d'uniformiser les mots à rechercher
# prend en argument le dataframe et la colonne (string) à mettre en minuscule
def string_lowercase(df,col):
    for i in range(len(df[col])):
        df[col][i] = df[col][i].lower()

# Retourne un dataframe avec pour une catégorie d'article, les mentions de chacune des drugs
# prend en argument une origine d'articles : pubmed ou clinical trials par exemple, et cette catégorie sous forme de string
def output_df(category,string_cat):
    info = [category["title"],category["journal"],string_cat]
    index = 0
    data = []
    output_col = ['drug', 'category', 'title', 'journal', 'date']

    for drug in drugs["drug"]:
        for art_title in info[0] :
            if drug in art_title :
                data.append([drug,info[2],art_title,category["journal"][index],category["date"][index]])
        index += 1

    return(pd.DataFrame(data, columns = output_col))

## lecture des données

drugs = pd.read_csv("data/drugs.csv")
pubmed = pd.read_csv("data/pubmed.csv")
pubmed_json = pd.read_json('data/pubmed.json')
clinical_trials = pd.read_csv("data/clinical_trials.csv")

# concaténation des 2 df pubmed
frames = [pubmed,pubmed_json]
pubmed = pd.concat(frames,ignore_index=True)

### Data pipeline

## data pre-processing

# uniformisation des dates, titres, et noms de colonnes
date_datetime_rename(pubmed,"date")
date_datetime_rename(clinical_trials,"date")
title_rename(pubmed,"title","journal")
title_rename(pubmed_json,"title","journal")
title_rename(clinical_trials,"scientific_title","journal")
string_lowercase(drugs,"drug")
string_lowercase(pubmed,"title")
string_lowercase(clinical_trials,"title")

## output
pubmed_output = output_df(pubmed,"pubmed")
clinical_trials = output_df(clinical_trials,"clinical_trials")
frames2 = [pubmed_output,clinical_trials]
result = pd.concat(frames2,ignore_index=True)

result_json = result.to_json(r'result.json')

# affichage du fichier JSON avec pandas
print(pd.read_json('result.json'))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
#  discussion_interface.py
#  Script that calls the course manager framework and performs various actions.
#  Arguments: <discussion tag>
import argparse
import sys

from config import Config, set_global_config
from discussion import Discussion

VALID_ACTIONS = [
    "make_discussion",
    "make_discussion_local",
    "make_answers",
    "make_answers_local"
]

def main(args):
    set_global_config(Config(True))
    disc = Discussion(args.discussion_tag)
    action = args.action
    if action == "make_discussion":
        disc.build_discussion()
        disc.push_discussion()
    elif action == "make_discussion_local":
        disc.build_discussion()
    elif action == "make_answers":
        disc.build_answers()
        disc.push_answers()
    elif action == "make_answers_local":
        disc.build_answers()
    else:
        assert False, "Should not reach here. Is an action in VALID_ACTIONS not implemented?"
    return 0

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("discussion_tag", metavar="DISCUSSION_TAG", help="Discussion tag (e.g. 4A, 5B, etc.)", type=str)
    parser.add_argument("action", metavar="ACTION", help="Valid action to perform", action="store", type=str,
                        choices=VALID_ACTIONS)
    args = parser.parse_args()
    sys.exit(main(args))


# May 2nd, 2022

file = open("input.txt", 'r')

alphabet = "abcdefghijklmnopqrstuvwxyz"
valid = set()
keys = {}
doors = {}
r = 0
for line in file:
	c = 0
	for symbol in line.strip():
		if symbol != '#':
			valid.add((r, c))
			if symbol in alphabet:
				keys[(r, c)] = symbol
			elif symbol in alphabet.upper():
				doors[(r, c)] = symbol.lower()
		c += 1
	r += 1

# Split maze
valid.remove((39, 40))
valid.remove((40, 39))
valid.remove((40, 40))
valid.remove((40, 41))
valid.remove((41, 40))

dist = [ { '@': {} }, { '@': {} }, { '@': {} }, { '@': {} } ]
for i in range(0, 4):
	for x in alphabet:
		dist[i][x] = {}
		for y in alphabet:
			dist[i][x][y] = -1
		dist[i][x][x] = 0

# Find keys required (doors) for each key
req = []
for i in range(0, 2):
	for j in range(0, 2):
		paths = [(40 + 2 * i - 1, 40 + 2 * j - 1, "")]
		memo = set([(40 + 2 * i - 1, 40 + 2 * j - 1)])
		depth = 0
		req.append({})
		while len(paths):
			depth += 1
			new = []
			for path in paths:
				r = path[0]
				c = path[1]
				d = path[2]
				for nxt in [(r+1,c),(r,c+1),(r-1,c),(r,c-1)]:
					if nxt in valid and nxt not in memo:
						t = d
						if nxt in doors:
							t += doors[nxt]
						elif nxt in keys:
							req[2 * i + j][keys[nxt]] = d
							dist[2 * i + j]['@'][keys[nxt]] = depth
							t += keys[nxt]
						new.append((nxt[0], nxt[1], t))
						memo.add((nxt[0], nxt[1]))
			paths = new

# Find distance between keys
for key in keys:
	q = 0
	for i in range(0, 4):
		if keys[key] in req[i]:
			q = i
			break
	paths = [(key[0], key[1], "")]
	memo = set([(key[0], key[1])])
	name = keys[key]
	need = set([x for x in req[q] if x != name])
	depth = 0
	while len(need):
		depth += 1
		new = []
		for path in paths:
			r = path[0]
			c = path[1]
			d = path[2]
			for nxt in [(r+1,c),(r,c+1),(r-1,c),(r,c-1)]:
				if nxt in valid and nxt not in memo:
					if nxt in keys and keys[nxt] in need:
						dist[q][name][keys[nxt]] = depth
						dist[q][keys[nxt]][name] = depth
						need.remove(keys[nxt])
					new.append((nxt[0], nxt[1], d))
					memo.add((nxt[0], nxt[1]))
		paths = new

minimum = 1000000000
paths = [(['@','@','@','@'], set([x for x in alphabet]), 0)]
while len(paths):
	new = []
	for path in paths:
		current = path[0]
		needed = path[1]
		depth = path[2]
		for nxt in needed:
			for i in range(0, 4):
				head = current[i]
				if nxt not in req[i]: continue
				possible = True
				for y in req[i][nxt]:
					if y in needed:
						possible = False
						break
				if possible:
					copy = set(needed)
					copy.remove(nxt)
					if len(copy) == 0:
						minimum = min(minimum, depth + dist[i][head][nxt])
					else:
						nxtCopy = [x for x in current]
						nxtCopy[i] = nxt
						new.append((nxtCopy, copy, depth + dist[i][head][nxt]))
				break
	paths = new

	# Remove duplicates
	i = 0
	while i < len(paths) - 1:
		j = i + 1
		while j < len(paths):
			if paths[i][0] == paths[j][0] and paths[i][1] == paths[j][1]:
				if paths[i][2] > paths[j][2]:
					paths[i] = paths[j]
				del paths[j]
			else:
				j += 1
		i += 1
print(minimum)


import argparse
import time
import shutil

import matplotlib.pyplot as plt

from os import path
from collections import OrderedDict

import torch
import torch.utils.data as data
import torch.optim as optim

import tensorboardX as tensorboard

from net.dataset import CODataset, BatchSampler, collate_fn, preprocessing, Transform
from net.LSTM import LSTM_RAE

from utils.meters import AverageMeter
import utils.logging as logging
from utils.snapshot import save_snapshot, resume_from_snapshot

NETWORK_INPUTS = ["signals", "labels"]
PAD_INPUTS =["signals"]

def make_parser():
    # ArgumentParser
    parser = argparse.ArgumentParser(description='PyTorch CNN Covision Test')

    parser.add_argument('--directory', metavar='EXPORT_DIR',
                        help='destination where trained network should be saved')

    parser.add_argument('--data', metavar='EXPORT_DIR',
                        help='destination where trained network should be saved')

    parser.add_argument('--log_dir', metavar='EXPORT_DIR')

    parser.add_argument("--local_rank", type=int)

    parser.add_argument("--epochs", type=int, default=100)

    parser.add_argument('--batch_size', metavar='N', type=int, default=1,
                        help='training and validation batch size ')

    parser.add_argument('--num_workers', metavar='N', type=int, default=2,
                        help='number of workers dataloader ')

    parser.add_argument("--log_interval", type=int, default=5)

    parser.add_argument("--val_interval", type=int, default=5)

    parser.add_argument("--learning_rate", type=float)

    parser.add_argument("--weight_decay", type=float)

    parser.add_argument("--refresh", type=bool, default=False)

    parser.add_argument("--resume", metavar="FILE", type=str, help="Resume training from given file")

    args = parser.parse_args()

    for arg in vars(args):
        print(' {}\t  {}'.format(arg, getattr(args, arg)))

    print('\n ')

    return parser

def log_debug(msg, *args, **kwargs):
    logging.get_logger().debug(msg, *args, **kwargs)

def log_info(msg, *args, **kwargs):
    logging.get_logger().info(msg, *args, **kwargs)

def make_dataloader(args):

    print("Creating dataloaders for dataset in %s", args.data)

    if args.refresh:
        preprocessing(path.join(args.directory, args.data))

    # Transform
    transform = Transform(path.join(args.directory, args.data))

    # Training dataloader

    train_db = CODataset(root_dir=path.join(args.directory, args.data),
                         split_name="train",
                         transform=transform)

    train_sampler = BatchSampler(train_db,
                                 batch_size=args.batch_size)

    train_dl = data.DataLoader(train_db,
                               batch_sampler=train_sampler,
                               pin_memory=True,
                               collate_fn=collate_fn,
                               num_workers=args.num_workers)

    # Validation dataloader
    val_db = CODataset(root_dir=path.join(args.directory, args.data),
                       split_name="val",
                       transform=transform)

    val_sampler = BatchSampler(val_db,
                               batch_size=args.batch_size)

    val_dl = data.DataLoader(val_db,
                             batch_sampler=val_sampler,
                             pin_memory=True,
                             collate_fn=collate_fn,
                             num_workers=args.num_workers)

    return train_dl, val_dl

def make_model(args, **varargs):
    seq_len = 32
    n_features = 1
    embedding_dim = 64
    batch_size = args.batch_size
    num_layers = 1

    return LSTM_RAE(seq_len, n_features, embedding_dim, batch_size, num_layers)

def make_optimizer(args, model):

    # Set-up optimizer hyper-parameters
    parameters = [
        {
            "params": model.parameters(),
            "lr": args.learning_rate,
        }
    ]

    # optimizer
    optimizer = optim.Adam(parameters)

    # scheduler
    scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=20, gamma=0.1)

    return optimizer, scheduler

def train(model, optimizer, scheduler, train_dataloader, meters, **varargs):

    model.train()
    optimizer.zero_grad()

    global_step = varargs["global_step"]

    data_time_meter = AverageMeter((), meters["loss"].momentum)
    batch_time_meter = AverageMeter((), meters["loss"].momentum)

    data_time = time.time()

    for it, batch in enumerate(train_dataloader):

        # Pad batch
        for k in PAD_INPUTS:
             batch[k].pad(max_size=1024)

        # Upload batch
        batch = {k: batch[k].cuda(device=varargs["device"], non_blocking=True) for k in NETWORK_INPUTS}

        data_time_meter.update(torch.tensor(time.time() - data_time))

        # Update scheduler
        global_step += 1
        batch_time = time.time()

        signals, signals_idx = batch["signals"].contiguous

        signal_pred = model(signals)

        loss = varargs["criterion"](signal_pred, signals)
        loss.backward()

        optimizer.step()
        optimizer.zero_grad()

        with torch.no_grad():
            meters["loss"].update(loss.cpu())
            #plt.plot(signals[0].cpu().numpy())

            #plt.plot(signal_pred[0].cpu().numpy())

            #plt.show()

        batch_time_meter.update(torch.tensor(time.time() - batch_time))

        # Clean-up
        del batch, loss

        # Log
        if varargs["summary"] is not None and (it + 1) % varargs["log_interval"] == 0:
            logging.iteration(
                varargs["summary"], "train", global_step,
                varargs["epoch"] + 1, varargs["num_epochs"],
                it + 1, len(train_dataloader),
                OrderedDict([
                    ("lr", scheduler.get_lr()[0]),
                    ("loss", meters["loss"]),
                    ("data_time", data_time_meter),
                    ("batch_time", batch_time_meter)
                ])
            )

        data_time = time.time()

    return global_step

def validate(model, val_dataloader, **varargs):
    model.eval()
    val_dataloader.batch_sampler.set_epoch(varargs["epoch"])

    loss_meter = AverageMeter(())
    data_time_meter = AverageMeter(())
    batch_time_meter = AverageMeter(())

    data_time = time.time()
    for it, batch in enumerate(val_dataloader):
        with torch.no_grad():

            # Pad batch
            for k in PAD_INPUTS:
                batch[k].pad(max_size=1024)

            # Upload batch
            batch = {k: batch[k].cuda(device=varargs["device"], non_blocking=True) for k in NETWORK_INPUTS}

            data_time_meter.update(torch.tensor(time.time() - data_time))

            batch_time = time.time()

            # Run network
            signals, signals_idx = batch["signals"].contiguous
            signal_pred = model(signals)

            loss = varargs["criterion"](signal_pred, signals)

            # Update meters
            loss_meter.update(loss.cpu())
            batch_time_meter.update(torch.tensor(time.time() - batch_time))

            del loss

            # Log batch
            if varargs["summary"] is not None and (it + 1) % varargs["log_interval"] == 0:
                logging.iteration(
                    None, "val", varargs["global_step"],
                    varargs["epoch"] + 1, varargs["num_epochs"],
                    it + 1, len(val_dataloader),
                    OrderedDict([
                        ("loss", loss_meter),
                        ("data_time", data_time_meter),
                        ("batch_time", batch_time_meter)
                    ])
                )

            data_time = time.time()

    return loss_meter.mean

def main(args):

    # Initialize
    device_id, device = args.local_rank, torch.device(args.local_rank)
    torch.cuda.set_device(device_id)

    # Initialize logging
    logging.init(args.log_dir, "training")
    summary = tensorboard.SummaryWriter(path.join(args.directory, args.log_dir))

    # Create dataloaders
    train_dataloader, val_dataloader = make_dataloader(args)

    # Create model
    model = make_model(args)

    # Init GPU stuff
    torch.backends.cudnn.enabled = False
    model = model.cuda(device=device)

    if args.resume:
        log_debug("Loading snapshot from %s", args.resume)
        snapshot = resume_from_snapshot(model, args.resume)

    # Create optimizer
    optimizer, scheduler = make_optimizer(args, model)

    #if args.resume:

    #    optimizer.load_state_dict(snapshot["state_dict"]["optimizer"])

    # criterion
    criterion = torch.nn.L1Loss(reduction='mean').to(device)
    #criterion = torch.nn.MSELoss(reduction='mean').to(device)

    # Training loop
    momentum = 1. - 1. / len(train_dataloader)
    meters = {
        "loss": AverageMeter((), momentum),
    }

    # loop params
    total_epochs = args.epochs

    if args.resume:
        starting_epoch = snapshot["training_meta"]["epoch"] +1
        best_score = snapshot["training_meta"]["best_score"]
        global_step = snapshot["training_meta"]["global_step"]

        for name, meter in meters.items():
            meter.load_state_dict(snapshot["state_dict"][name + "_meter"])
        del snapshot
    else:
        starting_epoch = 0
        best_score = 100
        global_step = 0

    for epoch in range(starting_epoch, total_epochs):
        log_info("Starting epoch %d", epoch + 1)

        scheduler.step(epoch)

        # Run training epoch
        global_step = train(model, optimizer, scheduler, train_dataloader, meters,
                            criterion=criterion,
                            device=device,
                            epoch=epoch,
                            summary=summary,
                            log_interval=args.log_interval,
                            num_epochs=total_epochs,
                            global_step=global_step)

        # Save snapshot (only on rank 0)
        snapshot_file = path.join(args.log_dir, "model_last.pth.tar")
        log_debug("Saving snapshot to %s", snapshot_file)

        meters_out_dict = {k + "_meter": v.state_dict() for k, v in meters.items()}

        save_snapshot(snapshot_file, epoch, 0, best_score, global_step,
                      model=model.state_dict(),
                      optimizer=optimizer.state_dict(),
                      **meters_out_dict)

        if (epoch + 1) % args.val_interval == 0:
            log_info("Validating epoch %d", epoch + 1)

            score = validate(model, val_dataloader,
                             criterion=criterion,
                             device=device,
                             epoch=epoch,
                             summary=summary,
                             log_interval=args.log_interval,
                             num_epochs=total_epochs,
                             global_step=global_step,
                             log_dir=args.log_dir)

            # Update the score on the last saved snapshot
            snapshot = torch.load(snapshot_file, map_location="cpu")
            snapshot["training_meta"]["last_score"] = score
            torch.save(snapshot, snapshot_file)

            del snapshot

            if score < best_score:
                best_score = score
                shutil.copy(snapshot_file, path.join(args.log_dir, "model_best.pth.tar"))

if __name__ == '__main__':

    parser = make_parser()

    main(parser.parse_args())


from argparse import ArgumentParser
from django.core.management.base import BaseCommand

from exams.modules.exams import ExamCreate

class Command(BaseCommand):
    """ Django cmd command for exam data uploading """
    help = 'Upload exam questions from file'

    def add_arguments(self, parser: ArgumentParser):
        """ Adds cmd arguments to command"""
        parser.add_argument('questions_file', nargs='?', type=str, help='Name of a JSON file with questions')
        parser.add_argument('--title', action='store', help='Title of an exam')
        parser.add_argument('--source', action='store', help='Exam source')

    def handle(self, *args, **options):
        """ Execute command """
        exam_title = options.get('title')
        exam_source = options.get('source')
        if not exam_title:
            exam_title = input('Enter the exam name: ')
        if not exam_source:
            exam_source = input('Enter the exam source: ')

        file_name = options.get('questions_file')
        file_with_questions = open(file_name, 'r')

        exam_create = ExamCreate()
        errors = exam_create.create_exam(exam_title, file_with_questions)
        if errors:
            for error in errors:
                print(error)


#!/usr/bin/env python3

import os
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import pyvista as pv
import matplotlib

#matplotlib.rcParams['font.sans-serif'] = "Comic Sans MS"
matplotlib.rc('font', serif='Arial')

#pv.set_plot_theme("default")
pv.set_plot_theme("document")
#pv.set_plot_theme("ParaView")

################################################################
# Load fault nodes for slip plot
################################################################
node_fname = 'fault2loc.txt'
node_data = pd.read_table(node_fname,header=0,sep='\s+')
inode = np.array(node_data['inode']).astype(int)

node_fname = 'plotNodes.dat'
node_data = pd.read_table(node_fname,sep='\s+',header=None)
node_data = node_data.T
node_data.columns = ['node']
node_id = node_data['node']

blocks = pv.read_exodus('mesh_4plots.exo')

print(blocks.keys())

args_grid = dict(label_font_size=50,title_font_size=50)

meshes = blocks['Element Blocks']

mantle_block = meshes[0]
crust_block = meshes[1]
fault_block = meshes[2]

faults = blocks['Side Sets'][2]

##################################################
# Plot simple rheology structure
##################################################

p = pv.Plotter(window_size=[1400,1400])
#p.set_background("31092D")
#p.add_mesh(meshes,show_edges=True,lighting=False,color='lightgoldenrodyellow') #, point_size=8.0, render_points_as_spheres=True)
p.add_mesh(mantle_block,show_edges=False,lighting=False,color='97bfc7',line_width=0.1)
p.add_mesh(crust_block,show_edges=False,lighting=False,color='b9b9b7',line_width=0.1)
p.add_mesh(fault_block,show_edges=False,lighting=False,color='e5b397',line_width=0.1)

p.add_mesh(faults,lighting=False,color='white',edge_color='red',show_edges=True,line_width=4)

#p.add_mesh(faults,show_edges=True,lighting=False,color='452142',line_width=1) #edge_color

p.show_bounds(grid='front', location='outer', all_edges=True,font_size=50,font_family='arial')
p.remove_scalar_bar()
p.enable_terrain_style()
p.show_axes()

side_cpos = [(-534.3440717049617, -1034.4191150136714, 952.1763261588785),
 (6.874786376953125, -4.4228363037109375, -249.99999999999545),
 (0.0, 0.0, 1.0)]

test = p.show(cpos=side_cpos,screenshot="model.png")
print(test)

##################################################
# Plot simple rheology structure with edges (aka wire frame)!
##################################################

p = pv.Plotter(window_size=[1400,1400])
#p.set_background("31092D")
#p.add_mesh(meshes,show_edges=True,lighting=False,color='lightgoldenrodyellow') #, point_size=8.0, render_points_as_spheres=True)
p.add_mesh(mantle_block,show_edges=True,lighting=False,color='97bfc7',line_width=0.1)
p.add_mesh(crust_block,show_edges=True,lighting=False,color='b9b9b7',line_width=0.1)
p.add_mesh(fault_block,show_edges=True,lighting=False,color='e5b397',line_width=0.1)

p.add_mesh(faults,lighting=False,color='white',edge_color='red',show_edges=True,line_width=4)

#p.add_mesh(faults,show_edges=True,lighting=False,color='452142',line_width=1) #edge_color

p.show_bounds(grid='front', location='outer', all_edges=True,font_size=50,font_family='arial')
p.remove_scalar_bar()
p.enable_terrain_style()
p.show_axes()

test = p.show(cpos=side_cpos,screenshot="model_mesh.png")
print(test)

##################################################
# Make fake legend
##################################################
plt.figure()
plt.plot([],[],'-',color='#97bfc7',linewidth=10,label='mantle block')
plt.plot([],[],'-',color='#b9b9b7',linewidth=10,label='crust block')
plt.plot([],[],'-',color='#e5b397',linewidth=10,label='fault block')
plt.plot([],[],'-',color='red',label='fault trace')

legend = plt.legend(frameon=False,ncol=4)
plt.gca().set_axis_off()

#fig.savefig("legend.png")

plt.savefig("model_legend.png", dpi=200)

os.system('convert model.png -trim model.png')
os.system('convert model_mesh.png -trim model_mesh.png')
os.system('convert model_legend.png -trim model_legend.png')
os.system('convert -border 70 -bordercolor white model_legend.png model_legend.png')


import socket
import time
import argparse

DEFAULT_PORT = 65432
DEFAULT_BUFFER_SIZE = 65536

def main(host, port, buf_size, rate_mbps):
    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
        s.connect((host, port))
        print(f"Connected to {host}:{port}")

        # Send the rate to the server
        s.sendall(rate_mbps.to_bytes(4, byteorder="big"))

        counter = 0
        start_time = time.time()

        try:
            while True:
                data = s.recv(buf_size)
                if not data:
                    break

                counter += len(data)

                current_time = time.time()
                if current_time - start_time >= 1:
                    speed_mbps = (
                        (counter * 8) / (1000 * 1000) / (current_time - start_time)
                    )
                    print(f"Current speed: {speed_mbps:.2f} Mbps")
                    counter = 0
                    start_time = current_time

        except KeyboardInterrupt:
            print("\nInterrupted by user")

        except Exception as e:
            print(f"Error: {e}")

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="TCP Brutal example client")
    parser.add_argument("host", help="Server host", type=str)
    parser.add_argument("rate_mbps", help="Rate in Mbps", type=int)
    parser.add_argument(
        "-p",
        "--port",
        help="Server port",
        type=int,
        default=DEFAULT_PORT,
    )
    parser.add_argument(
        "-b",
        "--buffer",
        help="Buffer size",
        type=int,
        default=DEFAULT_BUFFER_SIZE,
    )
    args = parser.parse_args()

    main(args.host, args.port, args.buffer, args.rate_mbps)


import math
import sys

from scipy.spatial.distance import pdist

from db_api import *

_DEBUG = False
_NEAREST_NUMBER = 15

def print_user_info(user_id):
    print(get_user_by_id(user_id))

def print_movie_info(movie_id):
    print(get_movie_by_id(movie_id))

def print_actual_rating(user_id, movie_id):
    print('The actual rating is', get_user_movie_rating(user_id, movie_id))

def average_rating(movie_id, silence=False):
    """average rating of movie"""
    mean_rating = statistics.mean([i.rating for i in get_user_who_rate_movie(movie_id)])
    if not silence:
        print('    Average Rating    '.center(80, '#'))
        print(round(mean_rating), '(', mean_rating, ')')
    return mean_rating

def custom_distance(a, b):
    """calculate distance between two list by custom way"""
    return sum([abs(i - j) ** 2 for (i, j) in zip(a, b)]) / len(a)

def cosine_distance(a, b):
    """calculate cosine distance for two list"""
    return pdist([a, b], 'cosine')

def canberra_distance(a, b):
    """calculate canberra distance for two list"""
    return pdist([a, b], 'canberra')

def count_info(neighbours):
    """print some distribution information for neighbours"""
    count_common = {}
    count_rating = {}
    for i in neighbours:
        if i[3] not in count_common:
            count_common[i[3]] = 1
        else:
            count_common[i[3]] += 1
        if i[1] not in count_rating:
            count_rating[i[1]] = 1
        else:
            count_rating[i[1]] += 1

    print(count_common)
    print(count_rating)
    print(statistics.mean([i[1] for i in neighbours]))

def similarity(candidate, user):
    """calculate the similarity of candidate based on ratings

    Args:
        candidate: dict {movie_id:rating}
        user: dict {movie_id:rating}

    Return:
        list: (user_id, rating, distance, common_movie)
    """
    candidate_rating_vector = []
    user_rating_vector = []
    for i in candidate:
        if i in user:
            candidate_rating_vector.append(candidate[i])
            user_rating_vector.append(user[i])

    ratio = math.log(30 + len(user_rating_vector), 64)
    return [candidate['user_id'], candidate['target_rating'],
            custom_distance(candidate_rating_vector, user_rating_vector) / ratio,
            len(user_rating_vector)]

def get_similar_user(user_id, movie_id):
    candidates = {}
    neighbours = []
    user_watched = {}

    current_user = {i.movie_id: i.rating for i in get_movie_rating_by_user(user_id)}
    if _DEBUG:
        print('This user has rated', len(get_movie_rating_by_user(user_id)), 'movies')

    user_watched = {i.user_id: i.movie_id for i in get_user_who_rate_movie(movie_id)}
    if _DEBUG:
        print(len(user_watched), 'users have watched this movie')

    for user in user_watched:
        # user who watched target movie
        if is_user_watched(user, list(current_user.keys())):
            # user who wathed target movie and has common movie with target user
            for rating in get_movie_rating_by_user(user):
                candidates.setdefault(rating.user_id, {})[rating.movie_id] = rating.rating
                candidates.setdefault(rating.user_id, {})['user_id'] = rating.user_id
                if rating.movie_id == movie_id:
                    candidates.setdefault(rating.user_id, {})['target_rating'] = rating.rating

    for candidate in candidates:
        neighbours.append(similarity(candidates[candidate], current_user))

    neighbours = list(filter(lambda x: not (x[2] == 0 and x[3] <= 5), neighbours))
    neighbours.sort(key=lambda x: x[2])

    if _DEBUG:
        count_info(neighbours[1:_NEAREST_NUMBER + 1])
    return neighbours[1:_NEAREST_NUMBER + 1]

def calculate_neighbour_rating(neighbours):
    """return predict score based on neighbour

    Args:
        neighbours: list (user_id, rating, distance, common_movie)

    Return:
        score: int
    """
    total_weight = 0
    total_score = 0
    if _DEBUG:
        for i in neighbours:
            print(i[1], i[2], i[3])
    for i, user in enumerate(neighbours):
        total_weight += 1 / math.log(2 + i, len(neighbours))
        total_score += user[1] / math.log(2 + i, len(neighbours))
    return total_score / total_weight

def nearest_neighbour(user_id, movie_id, silence=False):
    """use nearest neighbour to recommend"""
    neighbours = get_similar_user(user_id, movie_id)
    predict_value = calculate_neighbour_rating(neighbours)
    if not silence:
        print('    Nearest Neighbour    '.center(80, '#'))
        print(round(predict_value), '(', predict_value, ')')
    return predict_value

def get_movie_genres(movie_id):
    return get_movie_by_id(movie_id).genres.split('|')

def genres_similarity(movie_id, genres):
    count = 0
    for i in get_movie_genres(movie_id):
        if i in genres:
            count += 1
    return count

def parse_result(a):
    if a > 5.5:
        a = 5.5
    elif a < -0.5:
        a = -0.5
    return a

def slope_one(user_id, movie_id, silence=False):
    """use slope one algorithm to recommend"""
    watched_movies = [i.movie_id for i in get_movie_rating_by_user(user_id)]
    watched_movies.remove(movie_id)
    target_genres = get_movie_genres(movie_id)

    predict_ratings = []

    for movie in watched_movies:
        if genres_similarity(movie, target_genres):
            user_rating = get_user_movie_rating(user_id, movie)
            movie_rating, target_movie_rating, weight = get_two_movies_average_rating(movie, movie_id)
            if weight != 0:
                guess_rating = user_rating + movie_rating - target_movie_rating
                for i in range(weight):
                    predict_ratings.append(parse_result(guess_rating))
    if predict_ratings:
        predict_value = statistics.mean(predict_ratings)
    else:
        predict_value = average_rating(movie_id, True)

    if not silence:
        print('    Slope One    '.center(80, '#'))
        print(round(predict_value), '(', predict_value, ')')
    return predict_value

def hybrid_algorithm(avg, nearest, slope, silence=False):
    """hybrid algorithm based on average rating, nearest neighbour and slope one

    I don't use slope one because it's hard to find the similar movie, so the performance
    of slope one is poor.
    """
    sign = (nearest - avg) / abs(nearest - avg)
    ratio = 0.2
    predict_value = nearest + sign * abs(nearest - avg) * ratio
    if not silence:
        print('    Hybrid Algorithm    '.center(80, '#'))
        print(round(predict_value), '(', predict_value, ')')
    return predict_value

def predict(user_id, movie_id):
    """use different algorithm to predict movie rating"""
    print_user_info(user_id)
    print_movie_info(movie_id)
    print_actual_rating(user_id, movie_id)
    avg = average_rating(movie_id)
    nearest = nearest_neighbour(user_id, movie_id)
    slope = slope_one(user_id, movie_id)
    hybrid_algorithm(avg, nearest, slope)

def main():
    print('Predict movie rating')
    while True:
        try:
            user_id = int(input('\nPlease input user id:'))
            movie_id = int(input('please input movie id:'))
            predict(user_id, movie_id)
        except DoesNotExist:
            print('User/Movie does not exist, try again')

if __name__ == '__main__':
    sys.exit(main())


import os
import json
import copy
import shutil
import traceback
import pickle
import numpy as np
import pandas as pd

from datagen_args import *
from utils.datagen_utils import *
from blender_proc_scene import BlenderProcScene

def get_selected_objects(args):
    df = pd.read_csv(args.csv_file_path)
    unique_object_ids = df['objId'].unique()
    selected_object_indices = []
    # For each scene, choose object cateogry first
    for scene_num in range(args.num_scenes):
        num_object = np.random.randint(args.min_num_objects, args.max_num_objects+1, 1)[0]
        print("num_object: ", num_object)
        object_object_ids = np.random.choice(unique_object_ids, num_object, replace=True)
        selected_object_idx = []
        for obj_id in object_object_ids:
            obj_id_df = df[df['objId'] == obj_id]
            idx_in_obj_id_df = np.random.randint(0, len(obj_id_df), 1)[0]
            selected_object_idx.append(obj_id_df.iloc[idx_in_obj_id_df].name)
        selected_object_indices.append(selected_object_idx)

    selected_objects = []

    for selected_indices in selected_object_indices:
        selected_objects_i = []
        for idx in selected_indices:
            row = df.iloc[idx]
            selected_dict = {
                'synsetId' : row['synsetId'],
                'catId' : row['catId'],
                'ShapeNetModelId' : row['ShapeNetModelId'],
                'objId' : row['objId'],
                'half_or_whole' : row['half_or_whole'],
                'perch_rot_angle' : row['perch_rot_angle'],
            }
            selected_objects_i.append(selected_dict)
        selected_objects.append(selected_objects_i)
    return selected_objects

def generate_blender(args):
    if not args.single_object:
        selected_objects = get_selected_objects(args)
    else:
        selected_objects = []
        df = pd.read_csv(args.csv_file_path)
        synset_id_unique = [2876657, 2880940, 2946921, 3797390, 2942699, 3642806, 3593526, 3046257, 2773838]
        # df.synsetId.unique()
        # for synset_id in synset_id_unique:
        for synset_id in df['synsetId'].unique():
            df_select = df[df['synsetId'] == synset_id]
            for i in range(len(df_select)):
                row = df_select.iloc[i]
                selected_objects_i = [{
                    'synsetId' : row['synsetId'],
                    'catId' : row['catId'],
                    'ShapeNetModelId' : row['ShapeNetModelId'],
                    'objId' : row['objId'],
                    'half_or_whole' : row['half_or_whole'],
                    'perch_rot_angle' : row['perch_rot_angle'],
                }]
                selected_objects.append(selected_objects_i)

        # for i in range(len(df)):
        #     row = df.iloc[i]
        #     synset_id = '0{}'.format(row['synsetId'])
        #     if synset_id != '02876657':
        #         continue
        #     selected_objects_i = [{
        #         'synsetId' : row['synsetId'],
        #         'catId' : row['catId'],
        #         'ShapeNetModelId' : row['ShapeNetModelId'],
        #         'objId' : row['objId'],
        #         'half_or_whole' : row['half_or_whole'],
        #         'perch_rot_angle' : row['perch_rot_angle'],
        #     }]
        #     selected_objects.append(selected_objects_i)

    # print("len(selected_objects[0]): ", len(selected_objects[0]))
    all_yaml_file_name = []
    scene_num_to_selected = {}
    for scene_num in range(args.num_scenes):

        acc_scene_num = scene_num + args.start_scene_idx
        scene_folder_path = None
        if acc_scene_num >= len(selected_objects):
            continue
        try:
            scene_num_to_selected[acc_scene_num] = selected_objects[scene_num]
            blender_proc_scene = BlenderProcScene(acc_scene_num, selected_objects[scene_num], args)
            scene_folder_path = blender_proc_scene.scene_folder_path
            blender_proc_scene.output_yaml()
            if scene_num % 100 == 0:
                print(blender_proc_scene.yaml_fname)
            all_yaml_file_name.append(blender_proc_scene.yaml_fname)
        except:
            print('##################################### GEN Error!')
            if scene_folder_path is not None:
                shutil.rmtree(scene_folder_path)
            traceback.print_exc()

    # output to .sh file
    output_save_dir = os.path.join(args.scene_save_dir, args.train_or_test)
    sh_fname = os.path.join(output_save_dir, 'blender_proc.sh')
    output_fid = open(sh_fname, 'w+', encoding='utf-8')
    tmp_dir = os.path.join(args.scene_save_dir, 'tmp')
    for yaml_fname in all_yaml_file_name:
        line = f'python run.py {yaml_fname} --temp-dir {tmp_dir}\n'
        output_fid.write(line)
    output_fid.close()

    selected_object_file = os.path.join(
        output_save_dir,
        'selected_objects_{}_{}.pkl'.format(args.start_scene_idx, args.num_scenes),
    )
    with open(selected_object_file, 'wb+') as fh:
        pickle.dump(scene_num_to_selected, fh)

if __name__ == '__main__':
    generate_blender(args)


import os
import shutil
import traceback
import sys
from typing import List

import yaml

import spiders
from spiders import MovieInfo
from exts import log, get_config, file_scanner, get_codename, Config

def get_info(codename: str, config: Config) -> MovieInfo:
    db_sites: List[spiders.Spider] = list()
    for site in config.spider.resource_sites:
        spider_name = str.capitalize(site).replace(" ", "")
        db_sites.append(getattr(spiders, spider_name,
                               spiders.Javdb)())  # type: ignore
    info = None
    while info is None:
        try:
            site = db_sites.pop(0)
            site.set_proxies(config.spider.proxy)
            info = site.get_info(codename)
        except IndexError:
            info = MovieInfo()
        except:
            log(f'获取信息失败, 不再尝试: {codename}', 'ERROR')
            # info = MovieInfo()
            # raise

    return info

def set_mark(
        path: str,
        info: MovieInfo = None,  # type: ignore
        ignore_file: str = 'gavdener.ignore',
        info_file: str = 'info.yaml'):
    if os.path.isdir(path):
        target_dir = path
    else:
        target_dir = os.path.dirname(path)
    if not os.path.isdir(target_dir):
        log(f'停止添加标记, 目标路径未指向目录或文件: {path}', 'WARNING')
        return 1

    if info is None:
        filename = ignore_file
    else:
        filename = info_file
    target_path = os.path.join(target_dir, filename)
    with open(target_path, "w", encoding='utf-8') as fp:
        if info is None:
            log(f'标记为无效路径: {target_dir}')
        else:
            log(f'添加标记: {target_dir}')
            yaml_info = yaml.dump(info.to_dict(),
                                  indent=2,
                                  allow_unicode=True,
                                  sort_keys=False)
            fp.write(yaml_info)
    return 0

def move_movie(path: str, info: MovieInfo, config: Config) -> int:
    try:
        assert os.path.isfile(path)
        target_root_dir = config.general.target_dir
        main_actor = info.actors[0] if info.actors else info.default_text
        other_actors = info.actors[1:] if len(info.actors) > 1 else list()
        target_dir = os.path.join(target_root_dir, main_actor, info.codename)
        os.makedirs(target_dir, exist_ok=True)

        filename = os.path.basename(path)

        target_path = os.path.join(target_dir, filename)
        i = 0
        # 确保没有文件会被覆盖
        is_same = False
        while os.path.exists(target_path):
            log(f'文件已存在: {target_path}')
            if os.stat(path).st_ino == os.stat(target_path).st_ino:
                log(f'文件指向相同: {path} == {target_path}')
                is_same = True
                break
            root, ext = os.path.splitext(filename)
            i += 1
            tmp_name = f'{root}-{i}{ext}'
            target_path = os.path.join(target_dir, tmp_name)
        if config.general.debug:
            log(f'移动文件: {path} -> {target_path}', 'debug')
            set_mark(target_path,
                     info,
                     ignore_file=config.general.ignore_file,
                     info_file=config.general.info_file)
        else:
            log(f'移动文件: {path} -> {target_path}', 'info')
            if is_same:
                log(f'文件已存在, 仅补充信息: {target_path}')
                set_mark(target_path,
                         info,
                         ignore_file=config.general.ignore_file,
                         info_file=config.general.info_file)
            else:
                set_mark(target_path,
                         info,
                         ignore_file=config.general.ignore_file,
                         info_file=config.general.info_file)
                shutil.move(path, target_path)

        # 创建链接
        if config.scrapper.multi_actors:
            for actor in other_actors:
                tmp_dir = os.path.join(target_root_dir, actor, info.codename)
                os.makedirs(tmp_dir, exist_ok=True)
                tmp_path = os.path.join(tmp_dir, filename)
                if os.path.isfile(
                        target_path) and not os.path.exists(tmp_path):
                    log(f'创建链接: {tmp_path} -> {target_path}')
                    os.link(target_path, tmp_path)
                set_mark(tmp_path,
                         info,
                         ignore_file=config.general.ignore_file,
                         info_file=config.general.info_file)
        return 0
    except:
        log(traceback.format_exc(), "ERROR")
        return 1

def bar(msg):
    sys.stdout.write(f'\r{msg}'.ljust(128, " "))
    sys.stdout.flush()

def main(src_dir: str = None, config: str = None) -> int:  # type: ignore
    if config is None:
        _config = get_config()
    else:
        _config = get_config(config)

    if src_dir is None:
        src_dir = _config.general.media_dir

    all_movies = file_scanner(target_dir=src_dir,
                              include=_config.scrapper.target_exts,
                              ignore_file=_config.general.ignore_file)
    total = len(all_movies)
    log(f'共扫描到影片{total}部')

    for movie in all_movies:
        bar(f'正在处理: {movie} 进度: {all_movies.index(movie)+1}/{total}')
        log(f"开始处理: {movie}".rjust(128, ">"))
        codename = get_codename(movie, _config.general.info_file)
        if len(codename) <= 3:
            f'名称太短,已知信息不足,即将跳过: {codename}'
            continue
        try:
            log(f'获取信息: {codename}')
            info = get_info(codename, _config)
            log(f"影片信息:\n{info}")
            if info.codename and info.codename != MovieInfo.default_text:
                move_movie(movie, info, _config)
            else:
                set_mark(movie,
                         ignore_file=_config.general.ignore_file,
                         info_file=_config.general.info_file)
        except:
            log(f'处理失败: {movie}', 'ERROR')
            # log(traceback.format_exc(), 'ERROR')
            raise
        finally:
            log(f"处理结束: {movie}".rjust(128, "<"))

if __name__ == '__main__':
    if len(sys.argv) > 1:
        args = sys.argv[1:]
        main(*args)
    else:
        main()


# -*- coding: utf-8 -*-
"""
Created on Sat Oct 27 13:23:59 2018

@author: Akhil
"""
#    ----------------------------- TOKENIZATION ----------------------
from datetime import datetime # used to calculate the execution time
from nltk.tokenize import word_tokenize # used to divide strings into substrings
import re  # used to match or find other strings
import os,os.path
from collections import Counter # used to count hashable objects
import collections
import sys
from nltk.stem import PorterStemmer # used for stemming
from nltk.corpus import stopwords
from nltk.stem.wordnet import WordNetLemmatizer
import nltk
import operator
import pickle

lemmatizer = nltk.WordNetLemmatizer()
ps = PorterStemmer()

#DIR = 'E:/MS in UTD/Sem 1/Information Retrieval/Homework-2/cranfield/'
#DIR = '/people/cs/s/sanda/cs6322/Cranfield'
DIR = sys.argv[1]

list1 = [name for name in os.listdir(DIR) if os.path.isfile(os.path.join(DIR, name))] # list1 gets all the 1400 files which are in cranfield

l = []
d1={}
d2={}
docmaxtf = []
docLenlist = []
count1 = 0
docList = []
start1 = datetime.now()
for i in list1:
    count1 = count1 + 1
    filePath = DIR+i # relative path to the 1400 files
    file1 = open(filePath,"r")
    f1 = file1.read()

    rem = re.compile('<(.*?)>')
    k = re.sub(rem,'',f1)      # removes tags
    k = re.sub('[\d]+'," ",k) # removes all digits
    k = re.sub('[\W]+'," ",k) # removes all special characters
    k = word_tokenize(k) # tokenizes the documents
    k = [item.lower() for item in k] # converts Upper case to Lower case
    docLen = len(k)
    docList.append((count1,docLen))

    words_without_stopwords = [w for w in k if w not in stopwords.words('english')]

    # LEMMATIZER
    lemmas_list = []
    for var in words_without_stopwords:
        lemmas = lemmatizer.lemmatize(var)
        lemmas_list.append(lemmas)

    lc = collections.Counter(lemmas_list)
    mostFrequentTermLemma = lc.most_common(1)
    for key, value in mostFrequentTermLemma:
        max_tf_lemma = value
        docmaxtf.append((count1,max_tf_lemma))

    for term, termfreq in lc.items():
        abcd = [count1, termfreq,max_tf_lemma,docLen]
        dictlist=[]
        if term in d1:
            d1[term].append(abcd)
        else:
            dictlist.append(abcd)
            d1[term] = dictlist

dictLemma = collections.OrderedDict(sorted(d1.items()))

nof1 = 0
for term in dictLemma.items():
    nof1+=1

with open('Index_Version1_uncompressed.txt', 'w') as file:
    file.write("Term \t DF \t [DocId, TF, MaxTF, DocLen] \n")
    for term, tf in dictLemma.items():
        docFreq = str(len(tf))
        file.write(term+":\t")
        file.write(docFreq+"\t")
        file.write(str(tf))
        file.write("\n")

with open("Index_Version1_uncompressed.txt",'r') as txtfile:
    myString = txtfile.read()
ofile = open("Index_Version_1.uncompressed.bin",'wb')
pickle.dump(myString, ofile)
ofile.close()

end1 = datetime.now()

count2 = 0
start2 = datetime.now()
for i in list1:
    count2 = count2 + 1
    filePath = DIR+i # relative path to the 1400 files
    file1 = open(filePath,"r")
    f1 = file1.read()

    rem = re.compile('<(.*?)>')
    k = re.sub(rem,'',f1)      # removes tags
    k = re.sub('[\d]+'," ",k) # removes all digits
    k = re.sub('[\W]+'," ",k) # removes all special characters
    k = word_tokenize(k) # tokenizes the documents
    k = [item.lower() for item in k] # converts Upper case to Lower case
    docLen = len(k)

    words_without_stopwords = [w for w in k if w not in stopwords.words('english')]

    #STEMMING
    stems_list = []
    for w in words_without_stopwords:
        stems1 = ps.stem(w)
        stems1 = stems1.encode("utf-8")
        stems_list.append(stems1)

    lc1 = collections.Counter(stems_list)
    mostFrequentTermStems = lc1.most_common(1)
    for key, value in mostFrequentTermStems:
        max_tf_Stems = value

    for term, termfreq in lc1.items():
        abcde = [count2, termfreq,max_tf_Stems,docLen]
        dictlist_stems=[]
        if term in d2:
            d2[term].append(abcde)
        else:
            dictlist_stems.append(abcde)
            d2[term] = dictlist_stems

dictStems = collections.OrderedDict(sorted(d2.items()))

nof2 = 0
for term in dictStems.items():
    nof2+=1

with open('Index_Version2_uncompressed.txt', 'w') as file:
    file.write("Term \t DF \t [DocId, TF, MaxTF, DocLen] \n")
    for term, tf in dictStems.items():
        docFreq = str(len(tf))
        file.write(term+":\t")
        file.write(docFreq+"\t")
        file.write(str(tf))
        file.write("\n")

with open("Index_Version2_uncompressed.txt",'r') as txtfile:
    myString1 = txtfile.read()
ofile1 = open("Index_Version_2.uncompressed.bin",'wb')
pickle.dump(myString1, ofile1)
ofile1.close()

end2 = datetime.now()

start3 = datetime.now()
# Compressed index 1
termStringCompress1 = ""
for term, tf in dictLemma.items():
    termStringCompress1 = termStringCompress1 + str(len(term)) + term

termString = ""
with open('IndexVersion1Compressed.txt', 'w') as file:
    def gamma(number):
      binaryNum = str(bin(number))
      offset = binaryNum[3:]
      offsetLen = len(offset)
      length = unary(offsetLen)
      gammaString = str(length) + str(offset)
      return gammaString

    def unary(number):
        unaryString = ""
        i = 0
        while i < number:
            unaryString = unaryString + str(1)
            i += 1
        unaryString = unaryString + str(0)
        return unaryString

    termCounter = 0
    file.write("\nTerm String:\n"+termStringCompress1+"\n")
    for term, tf in dictLemma.items():
        termCounter = termCounter + 1
        listDocs = []
        listGaps = []
        for ele in tf:
            listDocs.append(ele[0])
        for ele in listDocs[:1]:
            listGaps.append(ele)

        for x in range((len(listDocs)-1)):
            listGaps.append( listDocs[x+1] - listDocs[x])
        binaryString = ""
        for ele in listGaps:
            gammaCode = str(gamma(ele))
            binaryString = binaryString + gammaCode
        file.write("\n"+binaryString+"\n")

        termString = termString + str(len(term)) + term
        if (termCounter % 8 == 0):
            index = len(termString)
            file.write("\n"+str(index)+"\n")

with open("IndexVersion1Compressed.txt",'r') as txtfile:
    myString2 = txtfile.read()
ofile2 = open("Index_Version_1.compressed.bin",'wb')
pickle.dump(myString2, ofile2)
ofile2.close()

end3 = datetime.now()

start4 = datetime.now()
# Compressed index 2
termStringCompress2 = ""
for term, tf in dictLemma.items():
    termStringCompress2 = termStringCompress1 + str(len(term)) + term

termString = ""
with open('IndexVersion2Compressed.txt', 'w') as file:
    def gamma(number):
      binaryNum = str(bin(number))
      offset = binaryNum[3:]
      offsetLen = len(offset)
      length = unary1(offsetLen)
      gammaString = str(length) + str(offset)
      return gammaString

    def unary1(number):
        unaryString = ""
        i = 0
        while i < number:
            unaryString = unaryString + str(1)
            i += 1
        unaryString = unaryString + str(0)
        return unaryString

    def delta(number):
        binaryNum = str(bin(number))
        offset = binaryNum[2:]
        offsetLen = len(offset)
        length = gamma(offsetLen)
        finalOffset = binaryNum[3:]
        deltaString = str(length) + str(finalOffset)
        return deltaString

    termCounter = 0
    file.write("\nTerm String:\n"+termStringCompress2+"\n")
    for term, tf in dictStems.items():
        termCounter = termCounter + 1
        listDocs = []
        listGaps = []
        for ele in tf:
            listDocs.append(ele[0])
        for ele in listDocs[:1]:
            listGaps.append(ele)

        for x in range((len(listDocs)-1)):
            listGaps.append( listDocs[x+1] - listDocs[x])
        binaryString = ""
        for ele in listGaps:
            deltaCode = str(delta(ele))
            binaryString = binaryString + deltaCode
        file.write("\n"+binaryString+"\n")
with open("IndexVersion2Compressed.txt",'r') as txtfile:
    myString3 = txtfile.read()
ofile3 = open("Index_Version_2.compressed.bin",'wb')
pickle.dump(myString3, ofile3)
ofile3.close()
end4 = datetime.now()

print("Elapsed time to build uncompressed Inverted index using lemmatization:", end1-start1)
print("Elapsed time to build uncompressed Inverted index using stemming:", end2-start2)
print("Elapsed time to build compressed Inverted index using lemmatization:", end3-start3)
print("Elapsed time to build compressed Inverted index using stemming:", end4-start4)

print("Size of index version1 :",os.path.getsize('Index_Version_1.uncompressed.bin'))
print("Size of index version2:",os.path.getsize('Index_Version_2.uncompressed.bin'))
print("Size of index version1 :",os.path.getsize('Index_Version_1.compressed.bin'))
print("Size of index version2:",os.path.getsize('Index_Version_2.compressed.bin'))

print("No. of inverted lists in index version1 uncompressed: ", nof1)
print("No. of inverted lists in index version2 uncompressed: ", nof2)
print("No. of inverted lists in index version1 compressed: ", nof1)
print("No. of inverted lists in index version2 compressed: ", nof2)

termsgiven = ["reynolds", "nasa", "prandtl", "flow", "pressure", "boundary", "shock"]
for term,value in dictLemma.items():
    if term in termsgiven:
        size1 = sys.getsizeof(value)
        docf = str(len(value))
        print("\n")
        print("Term: ", term)
        print("Doc freq:",docf)
        tf = 0
        for i in value:
            tf += i[1]
        print("Term frequency is: ", tf)
        print("Inverted list size: ", size1)

for term,value in dictLemma.items():
    if term == 'nasa':
        print("Term:",term)
        docf = str(len(value))
        print("Doc frequency for ", term ,"is: ",docf)
        countP = 0
        for i in value[:3]:
            countP+=1
            termFre = i[1]
            docLen = i[3]
            max_tf = i[2]
            print("\nNasa Posting list entry :", countP)
            print("Term Frequency: ",termFre)
            print("Document length: " , docLen)
            print("Max term frequency: " , max_tf)

dictIndex1 ={}
for term, tf in dictLemma.items():
    docFreq = len(tf)
    dictIndex1[term] = docFreq

maxValIndex1 = max(dictIndex1.values())
minValIndex1 = min(dictIndex1.values())
maxTermsIndex1 = []
minTermsIndex1 = []
for k, v in dictIndex1.items():
    if dictIndex1[k] == maxValIndex1:
        maxTermsIndex1.append(k)
    elif dictIndex1[k] == minValIndex1:
        minTermsIndex1.append(k)
print("\nTerms from index 1 with largest df:")
print(maxTermsIndex1)
print("\nTerms from index 1 with smallest df:")
print(minTermsIndex1)

dictIndex2 ={}
for term, tf in dictStems.items():
    docFreq = len(tf)
    dictIndex2[term] = docFreq

maxValIndex2 = max(dictIndex2.values())
minValIndex2 = min(dictIndex2.values())
maxTermsIndex2 = []
minTermsIndex2 = []
for k, v in dictIndex2.items():
    if dictIndex2[k] == maxValIndex2:
        maxTermsIndex2.append(k)
    elif dictIndex2[k] == minValIndex2:
        minTermsIndex2.append(k)
print("\nTerms(stems) from index 2 with largest df:")
print(maxTermsIndex2)
print("\nTerms(stems) from index 2 with smallest df:")
print(minTermsIndex2)

docMaxtfDict = dict(docmaxtf)
maxtfDocId = max(docMaxtfDict.items(), key=operator.itemgetter(1))[0]
print("\nDocument id with the largest max_tf in collection: %s" % maxtfDocId)

maxdocLengthDict = dict(docList)
maxDocLenDocId = max(maxdocLengthDict.items(), key=operator.itemgetter(1))[0]
print("\nDocument id with the largest doc length in collection: %s" % maxDocLenDocId)

os.remove('Index_Version1_uncompressed.txt')
os.remove('Index_Version2_uncompressed.txt')
os.remove('IndexVersion1Compressed.txt')
os.remove('IndexVersion2Compressed.txt')


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import pandas as pd
import numpy as np
import plotly.express as px
import seaborn as sns
from sklearn import linear_model
import matplotlib.pyplot as plt

# In[2]:

df = pd.DataFrame()
df = pd.read_csv("/Users/phonemyatkyaw/Downloads/world_population2022.csv")

# In[3]:

df.head()

# In[4]:

df.info()

# In[5]:

df.columns

# In[6]:

df.describe()

# In[7]:

df.isna().sum()

# In[8]:

df.rename(columns={"Country/Other":"Country", "Population (2020)":"Population"}, inplace=True)
df.rename(columns={"Urban Pop %": "Urban Pop"},inplace=True)

# In[9]:

avg_urban = df["Urban Pop"].mean()
avg_urban

# In[10]:

df["Urban Pop"] = df["Urban Pop"].fillna(avg_urban)

# In[11]:

df.isna().sum()

# In[12]:

df.head()

# In[13]:

highestpop = df.groupby("Country")["Population"].sum()

# In[14]:

highestpop= highestpop.sort_values(ascending=False)
highestpop

# In[15]:

highestpop = highestpop.to_frame()

# In[16]:

highestpop = highestpop.iloc[:5]
highestpop

# In[17]:

l = highestpop.index
e = [0.2,0,0,0,0]
c= ["#16F3AC","#F71C47","#D9F71C","#1E3DE8","#0BEAF5"]
plt.figure(figsize=(10,16))
plt.pie(highestpop["Population"],explode=e, colors= c, labels=l,autopct="%.2f%%")
plt.legend(title = "Top 5 Countries")
plt.title("Top 5 countries with highest population")
plt.savefig('highpop.png')

# In[18]:

lowpop = df.groupby("Country")["Population"].sum()
lowpop

# In[19]:

lowpop = lowpop.sort_values(ascending=True)
lowpop

# In[20]:

lowpop = lowpop.to_frame()

# In[21]:

lowpop = lowpop.iloc[:5]
lowpop

# In[22]:

l = lowpop.index
e = [0.2,0.2,0,0,0]
c= ["#16F3AC","#F71C47","#D9F71C","#1E3DE8","#0BEAF5"]
plt.figure(figsize=(10,16))
plt.pie(lowpop["Population"],colors=c, explode=e, labels=l,autopct="%.2f%%")
plt.legend(title = "Top 5 Countries with lowest population")
plt.title("Top 5 coutries with lowest population")
plt.savefig('lowpop.png')

# In[23]:

fig = px.choropleth(df, locations='Country',color='Yearly Change',locationmode='country names',color_continuous_scale='Electric',template='plotly_dark',title='Yearly Change')
fig.show()
plt.savefig('Yealy Change Map.png')

# In[24]:

fig = px.choropleth(df, locations='Country',color='Fert. Rate',locationmode='country names',color_continuous_scale='Electric',template='plotly_dark',title='World Map Based On Fert Rate')
fig.show()

# In[25]:

fig = px.choropleth(df, locations='Country',color='Migrants (net)',locationmode='country names',color_continuous_scale='Electric',template='plotly_dark',title='World Map Based on Migrants rate(net)')
fig.show()

# In[26]:

df.corr()

# In[27]:

plt.figure(figsize=(12,4))
sns.heatmap(df.corr(),annot=True, cmap="plasma", vmin=-1, vmax=1,linewidths=4)

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[28]:

subset = df[['Population', 'Yearly Change','Land Area (Km²)', 'Migrants (net)', 'Fert. Rate', 'World Share']]
mask = np.zeros_like(subset.corr())
triangle_indicies = np.triu_indices_from(mask)
mask[triangle_indicies] = True
plt.figure(figsize=(16,10))
sns.heatmap(subset.corr(), mask=mask, annot=True, annot_kws={"size": 10})
plt.style.use('dark_background')
plt.savefig('corr.png')

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[29]:

plt.figure(figsize=(10,5))
plt.subplot(3,1,1)
plt.title("Positive correlation between population & Net Change ")
sns.regplot(x="Net Change", y= "Population",data=df)
plt.savefig('pcorr.png')

# In[30]:

plt.figure(figsize=(10,5))
plt.subplot(3,1,1)
plt.title("Positive correlation between population & Land Area (Km²) ")
sns.regplot(x="Land Area (Km²)", y= "Population",data=df)
plt.savefig('pcorrl.png')

# In[31]:

sns.pairplot(df, vars=['Population', 'Land Area (Km²)', 'Urban Pop', 'World Share'],
             height=2.5)
plt.savefig('variable.png')

# In[32]:

df.columns

# In[33]:

df["Country"].count

# In[34]:

highmigrant = df.groupby("Country")["Migrants (net)"].sum()
highmigrant

# In[35]:

highmigrant = highmigrant.sort_values(ascending=True)
highmigrant

# In[36]:

highmigrant.to_frame()

# In[37]:

highmigrant = highmigrant.iloc[:5]
highmigrant

# In[38]:

l = highmigrant.index
data = highmigrant.values
plt.figure(figsize=(10,8))
plt.title("Top 5 Countries with highest migrant", color="Red", fontsize=22)
plt.ylabel("Country", color="Red", fontsize=22)
plt.xlabel("Migrants (net)", color="Red", fontsize=22)
plt.barh(l, width=data,);
plt.savefig('h5.png')

# In[39]:

lowmigrant = df.groupby("Country")["Migrants (net)"].sum()
lowmigrant

# In[40]:

lowmigrant = lowmigrant.sort_values(ascending=True)
lowmigrant

# In[41]:

lowmigrant.to_frame()
lowmigrant = lowmigrant[["Canada","United Kingdom","Turkey","Germany","United States"]]
lowmigrant

# In[42]:

l = lowmigrant.index
data = lowmigrant.values
plt.figure(figsize=(10,8))
plt.title("Top 5 Countries with lowest migrant", color="Red", fontsize=22)
plt.ylabel("Country", color="Red", fontsize=22)
plt.xlabel("Migrants (net)", color="Red", fontsize=22)
plt.barh(l, width=data)
plt.savefig('l5.png');

# In[43]:

df.columns

# In[44]:

df["Yearly Change"]

# In[45]:

plt.ylabel("Fert.Rate")
plt.xlabel("Med Age")
plt.scatter(df["Med. Age"],df["Fert. Rate"], color="g", marker="x")
plt.savefig('scatter for Fert and Med Age.png')

# In[46]:

X = df[["Med. Age"]]
y = df["Fert. Rate"]
model = linear_model.LinearRegression()
model.fit(X,y)
m = c.coef_[0]
model.predict([[30]])
b = model.intercept_
a= m* 30 +b
a

# In[ ]:

df.columns

# In[ ]:

plt.ylabel("Fert. Rate")
plt.xlabel("Med. Age")
plt.scatter(df["Med. Age"], df["Fert. Rate"], color="g", marker="x")
plt.plot(df["Med. Age"], model.predict(X))
plt.scatter (30, a,color = 'r', marker = "+", s=200)
plt.savefig("predict.png");

# In[ ]:

fig = px.choropleth(df, locations='Country',color='Migrants (net)',locationmode='country names',color_continuous_scale='Electric',template='plotly_dark',title='World Map Based on Migrants rate(net)')
fig.show()

# In[ ]:

fig = px.choropleth(df, locations='Country',color='Fert. Rate',locationmode='country names',color_continuous_scale='Electric',template='plotly_dark',title='World Map Based On Fert Rate')
fig.show()

# In[ ]:

# In[ ]:

fig = px.choropleth(df, locations='Country',color='Density (P/Km²)',locationmode='country names',color_continuous_scale='Electric',template='plotly_dark',title='Yearly Change')
fig.show()


# 2. Создать текстовый файл (не программно), сохранить в нем несколько строк,
# выполнить подсчет количества строк, количества слов в каждой строке.

line = " "
num_of_lines = 0
num_of_words = 0
eof = 0
try:
	with open("l05_hw2_text.txt", "r") as f:
		while not eof:
			line = f.readline()
			if line:
				num_of_lines += 1
				num_of_words = len(line.split())
				print(f"Line {num_of_lines} content {num_of_words} words")
			else:
				eof = 1
				print(f"Lines total: {num_of_lines}")

except:
	print(f"--Can't read file 'l05_wh2_text.txt'")


'''
Created on Feb 5, 2021

@author: janis
'''
from io import StringIO
import re
from .data import *
from . import FormattingError, MissingData
from typing import Callable
import pandas
import numpy
from . import Properties as _Properties
from sergio.sd.datasets.parsing import _get_default_class_name

class Symbol:
    def dumps(self, *args,**kwargs):
        s = StringIO()
        _ = self.dump(s, *args, **kwargs)
        return s.getvalue()

    def dump(self, s, *args,**kwargs):
        it = self.__generate__(*args, **kwargs)
        sz = 0
        try:
            while True:
                sz += s.write(next(it))
        except StopIteration:
            pass
        return sz

class DataSymbol(Symbol):
    def __init__(self, data=None):
        self._data = data if data is not None else _get_default_class_name(type(self))
    def __generate__(self, data):
        args, kwargs = consume(data, self._data)
        yield from self._generate(*args, **kwargs)
    @property
    def data(self): return self._data

class Value(DataSymbol):
    '''
    @param quote: Controls quoting.
        True quotes always,
        False never quotes
        'str' quotes strings,
        'name' quotes strings when necessary
    @param fmt Controls formatting.
        If a string, it follows the python formatting rules (e.q., "!s", ":.3f", etc).
        If callable, it is invoked with the value in question to yield the result.
        If a dict, the keys are classes mapping to formatting options. A key of None works as the default.
    @note Only doubles quotes are supported.
    '''

    def __init__(self, quote='str', fmt='!s', **kwargs):
        super().__init__(**kwargs)
        self._quote = quote
        self._fmt = fmt
    from . import rex_uquot
    def _generate(self, value):
        if value is None:
            raise MissingData(f'Required parameter {self.data} is of invalid type {value}.')
        fmt = self._fmt
        cls = type(value)
        if isinstance(fmt, dict):
            fn = fmt.get(cls, None)
            if fn is None:
                fn = fmt.get(None, '!s')
        else:
            fn = '!s' if fmt is None else fmt
        if isinstance(fn, str):
            text = f'{{{fn}}}'.format(value)
        elif isinstance(fn, Callable):
            text = fn(value)
        else:
            raise FormattingError(f'Unsupported format {fn} of type {type(fn).__name__} for value {value} of type {type(value).__name__}.')
        quote = self._quote
        if quote is True:
            must_quote = True
        elif quote is False:
            must_quote = False
        elif quote == 'str':
            must_quote = issubclass(cls, str)
        elif quote == 'name':
            must_quote = self.rex_uquot.match(text) is None
        else:
            raise FormattingError(f'Unsupported quoting option {quote} of type {type(quote).__name__}. Choose a boolean of ["str", "name"].')
        yield f'"{text}"' if must_quote else text

class Name(Value):
    def __init__(self, quote='name', **kwargs):
        super().__init__(quote=quote, **kwargs)

class Indices(DataSymbol):
    def __init__(self, base=0, data=None):
        super().__init__(data=data)
        self._base = base

    def _generate(self, indices):
        import numpy as np
        parts = []
        if len(indices)>0:
            diff = np.diff(indices)
            idx_non = np.where(diff!=1)[0]
            idx_chg = np.r_[idx_non,len(indices)]
            cnt_chg = np.diff(np.r_[0,idx_chg])
            beg = 0
            for end,cnt in zip(idx_chg, cnt_chg):
                offset = indices[beg] + self._base
                if cnt == 1:
                    part = str(offset)
                else:
                    part = f'{offset}-{offset+cnt-1}'
                beg = end
                parts.append(part)
        return ','.join(parts)

class Repeat(DataSymbol):

    def __init__(self, symbol, count=None, sep=' ', data=''):
        super().__init__(data=data)
        self._symbol = symbol
        self._count = count
        self._sep = sep

    def __generate__(self, data):
        sym = self._symbol
        sep = self._sep
        seq = sequence(data)
        if self._count is None:
            count = len(seq)
            if count is None:
                raise FormattingError(f'Unbounded repeat')
        else:
            count = self._count
        if count>0:
            yield from sym.__generate__(seq)
        for _ in range(1,count):
            if sep: yield sep
            yield from sym.__generate__(seq)

class Token(Symbol):
    '''
    @param unwrap For regular expression matches, returns the specific group.
    '''
    def __init__(self, text):
        self._text = text
    def __generate__(self, data):
        yield self._text
    def __str__(self):
        return f"{self._text!r}"

class Space(Token):
    def __init__(self):
        super().__init__(' ')

class Chain(Symbol):
    '''Parse in sequence all given symbols

    @param separate Controls consuming separators in between.
        False performs no consuming.
        True consumes whitespace
        Otherwise, the parameter is fed into rid.consume.
    '''
    def __init__(self, symbols, sep=' '):
        self._symbols = symbols
        self._sep = sep
    def __generate__(self, data):
        symbols = self._symbols
        sep = self._sep
        seq = sequence(data)
        try:
            if self._symbols:
                sym = symbols[0]
                yield from sym.__generate__(seq)
            for sym in symbols[1:]:
                if sep: yield sep
                yield from sym.__generate__(seq)
        except Exception as e:
            raise FormattingError(f'In {self} while generating {sym}.') from e

class Number(Symbol):
    def __init__(self, fmt='!s'):
        self._fmt = fmt
    def __generate__(self, data):
        yield f'{{0{self._fmt}}}'.format(data)

class Array(DataSymbol):
    '''Array symbol
    '''
    def __init__(self, quote=None, data=None):
        super().__init__(data=data)
        self._quote = quote

    def _generate(self, data):
        if data.dtype.name == 'str':
            sub = self.rex_escape.sub
            def fmt(value):
                escaped = sub(r'\"',value)
                return f'"{escaped}"'
        else:
            fmt = repr
        return ','.join(map(fmt, data))

class NewLine(str): pass

newline = NewLine('\n')

class LineChain(Chain):
    def __init__(self, symbols, sep=' ', end=newline):
        super().__init__(symbols, sep=sep)
        self._end = end

    def __generate__(self, *args, **kwargs):
        yield from super().__generate__(*args, **kwargs)
        yield self._end

class Header(Token):
    def __init__(self, name):
        super().__init__(f'@{name}')
        self.__name = name
    def __str__(self): return f'{self.__name}'

class Property(DataSymbol):
    def __init__(self, data='*'):
        super().__init__(data=data)
    SYMBOL = Chain((Name(),Token('='),Value(quote='name')),sep='')
    def _generate(self, pair):
        name, value = pair
        yield from self.SYMBOL.__generate__(props(name=name, value=value))

    def __str__(self): return self._name

class Properties(DataSymbol):
    def __init__(self, data=''):
        super().__init__(data=data)
    SYMBOL = Repeat(Property(data=''),sep=' ')
    def _generate(self, *pairs):
        yield from self.SYMBOL.__generate__(pairs)

class Payload(DataSymbol):
    def __init__(self, sparse=False, by_columns=False, dtype=None, data=None):
        super().__init__(data=data)
        self._dtype = dtype
        self._sparse = sparse
        self._by_columns = by_columns
    SYM_HDR = LineChain((Header('data'), Properties(data='*')))
    SYM_END = LineChain((Header('end'),Token('data')))

    class SparsePayloadLine(Symbol):
        SYMBOL = LineChain((Value(quote=False),Token(' ('), Indices(1),Token(') '),Array()),sep='')
        def __generate__(self, index, indices, values):
            yield from self.SYMBOL.__generate__(props(value=index+1, indices=indices, array=values))
    class DensePayloadLine(Symbol):
        SYMBOL = LineChain((Array(),))
        def __generate__(self, values):
            yield from self.SYMBOL.__generate__(props(array=values))

    def __generate__(self, payload):
        if self._by_columns is False:
            raise NotImplementedError('Can only write column-wise data.')

#        idl_sparse = np.r_[[isinstance(dtype, pd.SparseDtype) for dtype in data.dtypes]]
        if self._sparse:
            if isinstance(payload, pandas.DataFrame):
                def get_data(i):
                    arr = payload.iloc[:,i]
                    indices = arr.sp_index.to_int_index().indices
                    if len(indices) == 0:
                        return None
                    values = arr.sp_values
                    return (i, indices, values)
            else:
                raise NotImplementedError(f'No sparse matrices supported')
            sym = self.SparsePayloadLine()
        else:
            if isinstance(payload, pandas.DataFrame):
                get_data = lambda i: payload.iloc[:,i]
            else:
                get_data = lambda i: payload[:,i]
            sym = self.DensePayloadLine()
        num_data = payload.shape[1]
        mode = 'sparse' if self._sparse else 'dense'
        order = 'columns' if self._by_columns else 'rows'
        yield from self.SYM_HDR.__generate__(props(type=mode, order=order))
        for i in range(num_data):
            data = get_data(i)
            yield from sym.__generate__(data)
        yield from self.SYM_END.__generate__(pool(None))

class Matrix(DataSymbol):
    def __init__(self, sparse=None, by_columns=False, data='**'):
        super().__init__(data=data)
        self._by_columns = by_columns
        self._sym_data = Payload(sparse=sparse, by_columns=by_columns, dtype='int')
    SYM_META = LineChain((Header('matrix'), Name()))
    SYM_END = LineChain((Header('end'),Token('matrix'),Name()))
    def _generate(self, data, name):
        yield from self.SYM_META.__generate__(props(name=name))
        yield from self._sym_data.__generate__(data)
        yield from self.SYM_END.__generate__(props(name=name))

class Relation(DataSymbol):
    def __init__(self, data='**'):
        super().__init__(data=data)

    SYM_COLUMN = LineChain((Header('columns'), Repeat(Name(data=''))))
    ATTRIBUTE_TYPES = {'i': 'numeric', 'f':'numeric', 'U': 'string', 'S':'string'}
    STORAGE_TYPES = {'f': 'float', 'i': 'int'}
    def _generate_attr(self, kind, identifiers):
        try:
            attr_type = self.ATTRIBUTE_TYPES[kind]
        except KeyError:
            raise NotImplementedError(f'Kind {kind} is not supported.')
        storage_type = self.STORAGE_TYPES.get(kind)
        str_storage = '' if storage_type is None else f' storage={storage_type}'
        str_attr = f'@attribute {identifiers} {attr_type}{str_storage}'
        return str_attr
    SYM_ATTR = LineChain((Header('attribute'), Indices(1), Name(data='attr_type'), Properties(data='properties*')))
    def _generate_meta(self, df, name):
        yield from self.SYM_COLUMN.__generate__(sequence(df.columns))
        kinds = numpy.r_[[d.kind for d in df.dtypes]]
        for kind in numpy.unique(kinds):
            indices = numpy.where(kind == kinds)[0]
            try:
                attr_type = self.ATTRIBUTE_TYPES[kind]
            except KeyError:
                raise NotImplementedError(f'Kind {kind} is not supported.')
            storage_type = self.STORAGE_TYPES.get(kind)
            yield from self.SYM_ATTR.__generate__(props(indices=indices, attr_type=attr_type, properties=_Properties(storage=storage_type)))
    SYM_HDR = LineChain((Header('relation'), Name()))
    SYM_END = LineChain((Header('end'), Token('relation'), Name()))
    def _generate(self, data, name):
        yield from self.SYM_HDR.__generate__(props(data=data, name=name))
        yield from self._generate_meta(data, name)
        yield from Payload(by_columns=True).__generate__(data)
        yield from self.SYM_END.__generate__(props(name=name))

class DataSource(DataSymbol):
    def __init__(self, symbol, name, data='**'):
        super().__init__(data=data)
        self._symbol = symbol
        self._name = name

    SYMBOL = Name(data='')
    def _generate(self, name, data, manager, **kwargs):
        manager.add(name=name, symbol=self._symbol, data=data)
        yield from self.SYMBOL.__generate__(name)


import sys
import cv2

print('Hello, OpenCV', cv2.__version__)

# imread의 디폴트 값은 cv2.IMREAD_COLOR
img = cv2.imread('cat.bmp', cv2.IMREAD_GRAYSCALE)

if img is None:
    print('Image load failed!')
    sys.exit()

cv2.imwrite('cat_gray.png', img)

# 창 띄우기
cv2.namedWindow('image') # flags=None = windowAutoSize
# 영상 출력하기
cv2.imshow('image', img) # uint8 1바이트를 사용하는 ndarray를 넘겨주어야 한다.

# 여기 waitKey() 라는 함수까지 와야 실제로 영상이 출력이 된다.
# 실질적으로는 여기서 영상이 출력됌.
while True:
    if cv2.waitKey() == ord('q'):
        break
# key = cv2.waitKey(2000)   # 키보드 입력을 기다리는 함수.
# print(key)

# 창 닫기
cv2.destroyWindow('image')
# cv2.destroyAllWindows()


import os

import cv2
import numpy as np
from scipy.stats import mode, norm
from pytesseract import image_to_string, image_to_boxes
from PIL import Image

from progressbar import ProgressBar, Bar, SimpleProgress

from lib.region import Region
from lib.utils import plt_show, apply_canny

class TextDetection(object):

    def __init__(self, image_file, config, direction='both+', use_tesseract=True, details=False):

        ## Read image
        self.image_file = image_file
        img = cv2.imread(image_file)
        rgb_img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
        self.img = rgb_img
        self.h, self.w = img.shape[:2]

        self.direction = direction
        self.use_tesseract = use_tesseract
        self.details = details
        self.config = config
        self.AREA_LIM = config.AREA_LIM
        self.PERIMETER_LIM = config.PERIMETER_LIM
        self.ASPECT_RATIO_LIM = config.ASPECT_RATIO_LIM
        self.OCCUPATION_INTERVAL = config.OCCUPATION_INTERVAL
        self.COMPACTNESS_INTERVAL = config.COMPACTNESS_INTERVAL
        self.SWT_TOTAL_COUNT = config.SWT_TOTAL_COUNT
        self.SWT_STD_LIM = config.SWT_STD_LIM
        self.STROKE_WIDTH_SIZE_RATIO_LIM = config.STROKE_WIDTH_SIZE_RATIO_LIM
        self.STROKE_WIDTH_VARIANCE_RATIO_LIM = config.STROKE_WIDTH_VARIANCE_RATIO_LIM
        self.STEP_LIMIT = config.STEP_LIMIT
        self.KSIZE = config.KSIZE
        self.ITERATION = config.ITERATION
        self.MARGIN = config.MARGIN

        self.final = rgb_img.copy()

        self.height, self.width = self.img.shape[:2]

        self.gray_img = cv2.cvtColor(self.img.copy(), cv2.COLOR_RGB2GRAY)

        self.canny_img = apply_canny(self.img)

        self.sobelX = cv2.Sobel(self.gray_img, cv2.CV_64F, 1, 0, ksize=-1)
        self.sobelY = cv2.Sobel(self.gray_img, cv2.CV_64F, 0, 1, ksize=-1)

        self.stepsX = self.sobelY.astype(int)  ## Steps are inversed!! (x-step -> sobelY)
        self.stepsY = self.sobelX.astype(int)

        self.magnitudes = np.sqrt(self.stepsX * self.stepsX + self.stepsY * self.stepsY)
        self.gradsX = self.stepsX / (self.magnitudes + 1e-10)
        self.gradsY = self.stepsY / (self.magnitudes + 1e-10)

    def get_MSERegions(self, img):
        mser = cv2.MSER_create()
        regions, bboxes = mser.detectRegions(img)
        return regions, bboxes

    def get_stroke_properties(self, stroke_widths):
        if len(stroke_widths) == 0:
            return (0, 0, 0, 0, 0, 0)
        try:
            most_probable_stroke_width = mode(stroke_widths, axis=None)[0][0]
            most_probable_stroke_width_count = mode(stroke_widths, axis=None)[1][0]
        except IndexError:
            most_probable_stroke_width = 0
            most_probable_stroke_width_count = 0
        try:
            mean, std = norm.fit(stroke_widths)
            x_min, x_max = int(min(stroke_widths)), int(max(stroke_widths))
        except ValueError:
            mean, std, x_min, x_max = 0, 0, 0, 0
        return most_probable_stroke_width, most_probable_stroke_width_count, mean, std, x_min, x_max

    def get_strokes(self, xywh):

        x, y, w, h = xywh
        stroke_widths = np.array([[np.Infinity, np.Infinity]])
        for i in range(y, y + h):
            for j in range(x, x + w):
                if self.canny_img[i, j] != 0:
                    gradX = self.gradsX[i, j]
                    gradY = self.gradsY[i, j]

                    prevX, prevY, prevX_opp, prevY_opp, step_size = i, j, i, j, 0

                    if self.direction == "light":
                        go, go_opp = True, False
                    elif self.direction == "dark":
                        go, go_opp = False, True
                    else:
                        go, go_opp = True, True

                    stroke_width = np.Infinity
                    stroke_width_opp = np.Infinity
                    while (go or go_opp) and (step_size < self.STEP_LIMIT):
                        step_size += 1

                        if go:
                            curX = np.int(np.floor(i + gradX * step_size))
                            curY = np.int(np.floor(j + gradY * step_size))
                            if (curX <= y or curY <= x or curX >= y + h or curY >= x + w):
                                go = False
                            if go and ((curX != prevX) or (curY != prevY)):
                                try:
                                    if self.canny_img[curX, curY] != 0:
                                        if np.arccos(gradX * -self.gradsX[curX, curY] + gradY * -self.gradsY[curX, curY]) < np.pi / 2.0:
                                            stroke_width = int(np.sqrt((curX - i) ** 2  + (curY - j) ** 2))
                                            go = False
                                except IndexError:
                                    go = False

                                prevX = curX
                                prevY = curY

                        if go_opp:
                            curX_opp = np.int(np.floor(i - gradX * step_size))
                            curY_opp = np.int(np.floor(j - gradY * step_size))
                            if (curX_opp <= y or curY_opp <= x or curX_opp >= y + h or curY_opp >= x + w):
                                go_opp = False
                            if go_opp and ((curX_opp != prevX_opp) or (curY_opp != prevY_opp)):
                                try:
                                    if self.canny_img[curX_opp, curY_opp] != 0:
                                        if np.arccos(gradX * -self.gradsX[curX_opp, curY_opp] + gradY * -self.gradsY[curX_opp, curY_opp]) < np.pi/2.0:
                                            stroke_width_opp = int(np.sqrt((curX_opp - i) ** 2  + (curY_opp - j) ** 2))
                                            go_opp = False

                                except IndexError:
                                    go_opp = False

                                prevX_opp = curX_opp
                                prevY_opp = curY_opp

                    stroke_widths = np.append(stroke_widths, [(stroke_width, stroke_width_opp)], axis=0)

        stroke_widths_opp = np.delete(stroke_widths[:, 1], np.where(stroke_widths[:, 1] == np.Infinity))
        stroke_widths = np.delete(stroke_widths[:, 0], np.where(stroke_widths[:, 0] == np.Infinity))
        return stroke_widths, stroke_widths_opp

    def detect(self):
        res9 = np.zeros_like(self.img)
        if self.details:
            res0 ,res1, res2, res3, res4, res5, res6, res7, res8 = res9.copy(), res9.copy(), res9.copy(), \
                                                                   res9.copy(), res9.copy(), res9.copy(), \
                                                                   res9.copy(), res9.copy(), res9.copy()
        regions, bboxes = self.get_MSERegions(self.gray_img)
        #TODO regions, bboxes = self.get_MSERegions(self.img)

        n_mser_regions = len(regions)
        n_final_regions = 0
        if self.details:
            n1, n2, n3, n4, n5, n6, n7, n8, n9 = [0] * 9

        bar = ProgressBar(maxval=n_mser_regions, widgets=[Bar(marker='=', left='[', right=']'), ' ', SimpleProgress()])
        bar.start()

        for i, (region, bbox) in enumerate(zip(regions, bboxes)):
            bar.update(i + 1)

            region = Region(region, bbox)
            if self.details:
                res0 = region.color(res0)

            if region.area < self.w * self.h * self.AREA_LIM:
                continue
            if self.details:
                res1 = region.color(res1)
                n1 += 1

            if region.get_perimeter(self.canny_img) < (2 * (self.w + self.h) * self.PERIMETER_LIM):
                continue
            if self.details:
                res2 = region.color(res2)
                n2 += 1

            if region.get_aspect_ratio() > self.ASPECT_RATIO_LIM:
                continue
            if self.details:
                res3 = region.color(res3)
                n3 += 1

            occupation = region.get_occupation()
            if (occupation < self.OCCUPATION_INTERVAL[0]) or (occupation > self.OCCUPATION_INTERVAL[1]):
                continue
            if self.details:
                res4 = region.color(res4)
                n4 += 1

            compactness = region.get_compactness()
            if (compactness < self.COMPACTNESS_INTERVAL[0]) or (compactness > self.COMPACTNESS_INTERVAL[1]):
                continue
            if self.details:
                res5 = region.color(res5)
                n5 += 1

            x, y, w, h = bbox

            stroke_widths, stroke_widths_opp = self.get_strokes((x, y, w, h))
            if self.direction != "both+":
                stroke_widths = np.append(stroke_widths, stroke_widths_opp, axis=0)
                stroke_width, stroke_width_count, _, std, _, _ = self.get_stroke_properties(stroke_widths)
            else:
                stroke_width, stroke_width_count, _, std, _, _ = self.get_stroke_properties(stroke_widths)
                stroke_width_opp, stroke_width_count_opp, _, std_opp, _, _ = self.get_stroke_properties(stroke_widths_opp)
                if stroke_width_count_opp > stroke_width_count:        ## Take the stroke_widths with max of counts stroke_width (most probable one)
                    stroke_widths = stroke_widths_opp
                    stroke_width = stroke_width_opp
                    stroke_width_count = stroke_width_count_opp
                    std = std_opp

            if len(stroke_widths) < self.SWT_TOTAL_COUNT:
                continue
            if self.details:
                res6 = region.color(res6)
                n6 += 1

            if std > self.SWT_STD_LIM:
                continue
            if self.details:
                res7 = region.color(res7)
                n7 += 1

            stroke_width_size_ratio = stroke_width / max(region.w, region.h)
            if stroke_width_size_ratio < self.STROKE_WIDTH_SIZE_RATIO_LIM:
                continue
            if self.details:
                res8 = region.color(res8)
                n8 += 1

            stroke_width_variance_ratio = stroke_width / (std * std + 1e-10)
            if stroke_width_variance_ratio > self.STROKE_WIDTH_VARIANCE_RATIO_LIM:
                n_final_regions += 1
                res9 = region.color(res9)
                if self.details:
                    n9 += 1

        bar.finish()
        print("{} regions left.".format(n_final_regions))

        ## Binarize regions
        binarized = np.zeros_like(self.gray_img)
        rows, cols, _ = np.where(res9 != [0, 0, 0])
        binarized[rows, cols] = 255

        ## Dilate regions and find contours
        kernel = np.zeros((self.KSIZE, self.KSIZE), dtype=np.uint8)
        kernel[(self.KSIZE // 2)] = 1

        res = np.zeros_like(self.gray_img)
        dilated = cv2.dilate(binarized.copy(), kernel, iterations=self.ITERATION)
        _, contours, hierarchies = cv2.findContours(dilated, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

        if self.use_tesseract:
            print("Tesseract will eliminate..")
            temp_file = 'text.jpg'
        for i, (contour, hierarchy) in enumerate(zip(contours, hierarchies[0])):
            if hierarchy[-1] != -1:
                continue

            if self.use_tesseract:
                x, y, w, h = cv2.boundingRect(contour)
                if (y - self.MARGIN > 0) and (y + h + self.MARGIN < self.height) and (x - self.MARGIN > 0) and (x + w + self.MARGIN < self.width):
                    cv2.imwrite(temp_file, self.final[y - self.MARGIN:y + h + self.MARGIN, x - self.MARGIN:x + w + self.MARGIN])
                else:
                    cv2.imwrite(temp_file, self.final[y:y + h, x:x + w])

                ## Run tesseract
                string = image_to_string(Image.open(temp_file))
                if string is not u'':
                    rect = cv2.minAreaRect(contour)
                    box = cv2.boxPoints(rect)
                    box = np.int0(box)
                    cv2.drawContours(self.final, [box], 0, (0, 255, 0), 2)
                    cv2.drawContours(res, [box], 0, 255, -1)
                os.remove(temp_file)

            else:
                rect = cv2.minAreaRect(contour)
                box = cv2.boxPoints(rect)
                box = np.int0(box)
                cv2.drawContours(self.final, [box], 0, (0, 255, 0), 2)
                cv2.drawContours(res, [box], 0, 255, -1)

        if self.details:
            plt_show((self.img, "Image"), \
                     (self.canny_img, "Canny"), \
                     (res0, "MSER,({} regions)".format(n_mser_regions)), \
                     (res1, "Min Area={},({} regions)".format(self.AREA_LIM, n1)), \
                     (res2, "Min Perimeter={},({} regions)".format(self.PERIMETER_LIM, n2)), \
                     (res3, "Aspect Ratio={},({} regions)".format(self.ASPECT_RATIO_LIM, n3)), \
                     (res4, "Occupation={},({} regions)".format(self.OCCUPATION_INTERVAL, n4)), \
                     (res5, "Compactness={},({} regions)".format(self.COMPACTNESS_INTERVAL, n5))
                    )

            plt_show((res6, "STROKES TOTAL COUNT={},({} regions)".format(self.SWT_TOTAL_COUNT, n6)),\
                     (res7, "STROKES STD={},({} regions)".format(self.SWT_STD_LIM, n7)), \
                     (res8, "STROKE/SIZE RATIO={},({} regions)".format(self.STROKE_WIDTH_SIZE_RATIO_LIM, n8)),\
                     (res9, "STROKE/VARIANCE RATIO={},({} regions)".format(self.STROKE_WIDTH_VARIANCE_RATIO_LIM, n9)),\
                     (binarized, "Binarized"),\
                     (dilated, "Dilated (iterations={},ksize={})".format(self.ITERATION, self.KSIZE)),\
                     (self.final, "Final")\
                    )

        return res

    def full_OCR(self):
        bounded = self.img.copy()
        res = np.zeros_like(self.gray_img)

        string = image_to_string(Image.open(self.image_file))
        if string == u'':
            return bounded, res

        boxes = image_to_boxes(Image.open(self.image_file))
        boxes = [map(int, i) for i in [b.split(" ")[1:-1] for b in boxes.split("\n")]]

        for box in boxes:
            b = (int(box[0]), int(self.h - box[1]), int(box[2]), int(self.h - box[3]))
            cv2.rectangle(bounded, (b[0], b[1]), (b[2], b[3]), (0, 255, 0), 2)
            cv2.rectangle(res, (b[0], b[1]), (b[2], b[3]), 255, -1)

        return bounded, res