"""Conftest file"""
from datetime import datetime, timedelta
from unittest.mock import patch
from uuid import uuid4

import jwt
import pytest
from fastapi.testclient import TestClient
from pytest_mock import MockerFixture

from main import app
from modules.database.models import Badge, User
from modules.utilities.auth import SECRET_KEY
from modules.utilities.database import SessionLocal

@pytest.fixture(scope="module")
def client():
    """
    Create a FastAPI test client using the app fixture.
    """
    return TestClient(app=app)

@pytest.fixture
def mock_get():
    """Mock get request"""
    with patch("requests.get") as mock:
        yield mock

@pytest.fixture
def mock_redis():
    """Mock redis"""
    with patch("redis.Redis") as mock:
        yield mock.return_value

@pytest.fixture
def db_session():
    """
    Create a new database session for each test.
    """
    test_db_session = SessionLocal()
    yield test_db_session
    test_db_session.close()

@pytest.fixture
def mock_generate_jwt_token(mocker: MockerFixture):
    """
    Mock the generate_jwt_token function.
    """
    return mocker.patch(
        "modules.utilities.auth.generate_jwt_token",
        return_value="mocked_token",
    )

@pytest.fixture
def mock_customer_alias():
    """Test customer alias"""
    return "bbg"

@pytest.fixture
# pylint: disable=W0621
def mock_add_badge_user(db_session):
    """Create mock user for adding badges"""

    new_test_user = User(id=uuid4(), customer_id="bbg")
    db_session.add(new_test_user)
    db_session.commit()

    return new_test_user

@pytest.fixture
# pylint: disable=W0621
def mock_update_badge_user(db_session):
    """Create mock user for updating badges"""

    mock_user = User(id=uuid4(), customer_id="xbahn")
    badge1 = Badge(badge_name="SPAMMER", user=mock_user)
    db_session.add(mock_user)
    db_session.add_all([badge1])
    db_session.commit()
    return mock_user

@pytest.fixture
# pylint: disable=W0621
def mock_delete_badge_user(db_session):
    """Create mock user for deleting badges"""
    mock_user = User(id=uuid4(), customer_id="xbahn")
    badge1 = Badge(badge_name="SPAMMER", user=mock_user)
    badge2 = Badge(badge_name="CONTRIBUTOR", user=mock_user)

    db_session.add(mock_user)
    db_session.add_all([badge1, badge2])
    db_session.commit()
    return mock_user

@pytest.fixture
def generate_mock_token():
    """Generate test token"""
    token_payload = {
        "customer_alias": "xbahn",
        "exp": datetime.utcnow() + timedelta(minutes=30),
    }
    test_token = jwt.encode(token_payload, SECRET_KEY, algorithm="HS256")
    return f"Bearer {test_token}"


#!/usr/bin/env python3

# https://www.hackerrank.com/challenges/acm-icpc-team

n, m = input().strip().split(' ')
n, m = [int(n), int(m)]
topic = []
topic_i = 0
for topic_i in range(n):
    topic_t = str(input().strip())
    topic.append(int(topic_t, 2))

mmax = mcount = 0
for i in range(n):
    t = topic.pop()
    for r in topic:
        bn = r | t
        cur = bin(bn).count('1')
        if cur > mmax:
            mmax = cur
            mcount = 1
        elif cur == mmax:
            mcount += 1
print(str(mmax))
print(str(mcount))


#!/usr/bin/python

import sys
import email
import email.utils

def main():
    msg = email.message_from_file(sys.stdin)

    _, toaddr = email.utils.parseaddr(msg['To'])
    fromname, _ = email.utils.parseaddr(msg['From'])

    if 'reply-isp-' in toaddr:
        del msg['From']
        newfrom = toaddr.replace('reply-isp-', 'reply-', 1)
        msg['From'] = email.utils.formataddr((fromname, newfrom))
    elif 'reply-' in toaddr:
        del msg['From']
        newfrom = toaddr.replace('reply-', 'reply-isp-', 1)
        msg['From'] = email.utils.formataddr((fromname, newfrom))
    else:
        # do nothing
        pass

    sys.stdout.write(msg.as_string())

if __name__ == '__main__':
    try:
        main()
    except Exception as exc:
        print >>sys.stderr, "Error: {}".format(exc)
        sys.exit(1)


from django.shortcuts import render, redirect
from django.contrib.auth import logout, login
from oauth.forms import RegistrationUserForm, AuthenticationUserForm

def register_view(request):
    if request.method == "POST":
        form = RegistrationUserForm(request.POST)
        if form.is_valid():
            user = form.save()
            login(request, user)
            return redirect("home")

    else:
        form = RegistrationUserForm()
    return render(request, 'register.html', {'form': form})

def login_view(request):
    if request.method == "POST":
        form = AuthenticationUserForm(data=request.POST)
        if form.is_valid():
            user = form.get_user()
            login(request, user)
            return redirect("home")

    else:
        form = AuthenticationUserForm()
        return render(request, 'login.html', {'form': form})

def logout_user(request):
    logout(request)
    return redirect('login')

def home_view(request):
    return render(request, 'home.html')


# 2017年10月29日20:51:58
# 本程序的功能是：测试‘异常’处理机制，以及统计文件的字符数

try:
    f = open('E:\\something1.txt')
    temp = f.read()
except FileNotFoundError:
    msg = print('The File '+'E:\\something1.txt'+' was not found !')
else:
    words = temp.split()#【注意】split是按照空格进行划分字符串
    num_words = len(words)
    print('The File '+'E:\\something1.txt'+' have '+str(num_words)+' words.')
    f.close()


from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.core.paginator import Paginator
from django.shortcuts import get_object_or_404, redirect, render

from .forms import CommentForm, PostForm
from .models import Follow, Group, Post, User

def post_peginator(request, posts_list):
    paginator = Paginator(posts_list, settings.POSTS_PER_PAGE)
    page_number = request.GET.get('page')
    return paginator.get_page(page_number)

def index(request):
    posts_list = Post.objects.select_related('author').all()
    page_obj = post_peginator(request, posts_list)
    context = {
        'page_obj': page_obj,
    }
    return render(request, 'posts/index.html', context)

def group_posts(request, slug):
    group = get_object_or_404(Group, slug=slug)
    posts_list = Post.objects.select_related('author').filter(group=group)
    page_obj = post_peginator(request, posts_list)
    context = {
        'page_obj': page_obj,
        'group': group,
    }
    return render(request, 'posts/group_list.html', context)

def profile(request, username):
    author = get_object_or_404(User, username=username)
    posts_list = Post.objects.filter(author=author)
    page_obj = post_peginator(request, posts_list)
    following = (
        request.user.is_authenticated
        and Follow.objects.filter(
            user=request.user,
            author=author,
        ).exists()
    )
    context = {
        'page_obj': page_obj,
        'author': author,
        'following': following,
    }
    return render(request, 'posts/profile.html', context)

def post_detail(request, post_id):
    posts = get_object_or_404(Post, id=post_id)
    posts_list = Post.objects.select_related('author').filter(
        author=posts.author
    )
    comments = posts.comments.all()
    form = CommentForm()
    page_obj = post_peginator(request, posts_list)
    context = {
        'form': form,
        'posts': posts,
        'page_obj': page_obj,
        'comments': comments,
    }
    return render(request, 'posts/post_detail.html', context)

@login_required
def post_create(request):
    form = PostForm(
        request.POST or None,
        files=request.FILES or None,
    )
    author = get_object_or_404(User, pk=request.user.id)
    if form.is_valid():
        post = form.save(commit=False)
        post.author = request.user
        post.save()
        return redirect('posts:profile', author)
    context = {
        'form': form,
    }
    return render(request, 'posts/create_post.html', context)

@login_required
def post_edit(request, post_id):
    post = get_object_or_404(Post, pk=post_id, author=request.user)
    form = PostForm(
        request.POST or None,
        files=request.FILES or None,
        instance=post,
    )
    if form.is_valid():
        post = form.save(commit=False)
        post.save()
        return redirect('posts:post_detail', post_id)
    context = {
        'post': post,
        'form': form,
        'editing': True,
    }
    return render(request, 'posts/create_post.html', context)

@login_required
def add_comment(request, post_id):
    post = get_object_or_404(Post, id=post_id)
    form = CommentForm(request.POST or None)
    if form.is_valid():
        comment = form.save(commit=False)
        comment.author = request.user
        comment.post = post
        comment.save()
        return redirect('posts:post_detail', post_id=post_id)
    context = {
        'form': form,
    }
    return render(request, 'posts/post_detail.html', context)

@login_required
def follow_index(request):
    posts_list = Post.objects.select_related('author').filter(
        author__following__user=request.user
    )
    page_obj = post_peginator(request, posts_list)
    context = {
        'page_obj': page_obj,
    }
    return render(request, 'posts/follow.html', context)

@login_required
def profile_follow(request, username):
    author = get_object_or_404(User, username=username)
    follow = Follow.objects.filter(user=request.user, author=author)
    if request.user != author and not follow.exists():
        Follow.objects.create(user=request.user, author=author)
    return redirect('posts:profile', author)

@login_required
def profile_unfollow(request, username):
    author = get_object_or_404(User, username=username)
    follow = Follow.objects.filter(
        user=request.user,
        author=author,
    )
    if follow.exists():
        follow.delete()
    return redirect('posts:profile', author)


import os
from os.path import join, dirname
from dotenv import load_dotenv
from pyspark.sql import SparkSession
from pyspark.sql.functions import col, when, row_number, last, count
from pyspark.sql.window import Window
from fuzzywuzzy import fuzz
import pymongo
import boto3
import re

# load variables from .env file
dotenv_path=join(dirname(__file__),"dags/.env")
load_dotenv(dotenv_path=dotenv_path)

# Accessing environment variables
AWS_ACCESS_KEY=os.getenv('AWS_ACCESS_KEY')
AWS_SECRET_KEY=os.getenv('AWS_SECRET_KEY')
CRM_PATH=os.getenv('CRM_PATH')
DEVICE_PATH=os.getenv('DEVICE_PATH')

# Creating singleton class ,so that it creates only one spark instance
class SingletonSparkSession:
    """
        Creates a SparkSession with necessary configurations.

        Returns:
            SparkSession: The created SparkSession object.
    """

    _instance = None

    def __new__(cls):
        if not cls._instance:
            cls._instance = super().__new__(cls)
            cls._instance.spark = SparkSession.builder \
                .appName("read_S3_write_mongodb") \
                .config("spark.jars.packages", "org.mongodb.spark:mongo-spark-connector_2.12:3.0.1,org.apache.hadoop:hadoop-aws:3.2.2,com.amazonaws:aws-java-sdk-bundle:1.12.180") \
                .config("spark.hadoop.fs.s3a.aws.credentials.provider", "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider") \
                .config("spark.driver.memory", "15g").config("spark.driver.maxResultSize", "4g") \
                .config("spark.hadoop.fs.s3a.access.key", AWS_ACCESS_KEY)\
                .config("spark.hadoop.fs.s3a.secret.key", AWS_SECRET_KEY)\
                .getOrCreate()
        return cls._instance.spark

# Writing the data to moongodb
class SingletonMongoWriter:
    """
    Singleton class for writing data to MongoDB.
    """
    _instance = None

    def __new__(cls):
        """
        Create and return a new instance of the class if it doesn't exist.

        Returns:
            SingletonMongoWriter: The instance of the class.
        """

        if not cls._instance:
            cls._instance = super().__new__(cls)
        return cls._instance

    def write_to_Mongodb(self, dataframe, output_uri):
        """
        Write a DataFrame to MongoDB.

        Args:
            dataframe (pyspark.sql.DataFrame): The DataFrame containing the data to be written.
            output_uri (str): The MongoDB connection URI specifying the target database and collection.

        Returns:
            None
        """
        dataframe.write.format("com.mongodb.spark.sql.DefaultSource") \
            .mode("append") \
            .option("spark.mongodb.output.uri", output_uri) \
            .save()

# For validating data
class DataProcessor:
    """
    A class that validates and processes data.

    This class provides methods for checking file existence in an S3 bucket,
    performing fuzzy matching on words to determine gender, and processing
    CRM and device data.

    Attributes:
        spark (SparkSession): The singleton SparkSession object.
        mongo_writer (SingletonMongoWriter): The singleton MongoDB writer object.

    Methods:
        check_file_exists_in_S3(bucket_name, key):
            Checks if a file exists in the specified S3 bucket.

        fuzzy_ratio(word):
            Determines the gender based on a fuzzy matching ratio of the input word.

        process_crm_data(crm_s3_path):
            Processes CRM data from an S3 bucket.

        process_device_data(device_s3_path):
            Processes device data from an S3 bucket.
    """
    def __init__(self):
        """
        Initializes the DataProcessor object.

        It creates instances of the SingletonSparkSession and SingletonMongoWriter
        classes to access the SparkSession and MongoDB writer objects.
        """
        self.spark = SingletonSparkSession()
        self.mongo_writer = SingletonMongoWriter()

    def check_file_exists_in_S3(self, bucket_name, key):
        """
        Checks if a file exists in the specified S3 bucket.

        Args:
            bucket_name (str): The name of the S3 bucket.
            key (str): The key or path of the file in the S3 bucket.

        Returns:
            bool: True if the file exists in the S3 bucket, False otherwise.
        """

        # Create a session using AWS access key and secret key
        session = boto3.Session(aws_access_key_id=AWS_ACCESS_KEY, aws_secret_access_key=AWS_SECRET_KEY)

        # Create an S3 client using the session
        s3_client = session.client('s3')

        try:
            # Use the head_object method to check if the file exists
            s3_client.head_object(Bucket=bucket_name, Key=key)
            print(f"The file '{key}' exists in the bucket '{bucket_name}'.")
            return True

        except s3_client.exceptions.ClientError as e:
            if e.response['Error']['Code'] == '404':
                # If the file does not exist, handle the 404 error
                print(f"The file '{key}' does not exist in the bucket '{bucket_name}'.")
            else:
                # Handle other exceptions
                print(f"We got an exception: {e}")

            return False

    @staticmethod
    def fuzzy_ratio(word):
        """
        Determines the gender based on a fuzzy matching ratio of the input word with 'male' and 'female'.

        Args:
            word (str): The input word.

        Returns:
            str or None: The gender ('male' or 'female') if a match is found with a sufficient ratio, or None otherwise.
        """

        if word is None:
            return None

        male_pattern = "ma(l)*(e)*"
        female_pattern = "fe(m)*(a)*(l)*(e)*"

        # Convert the word to lowercase for case-insensitive matching
        word = word.lower()

        # Check if the word directly matches 'f' or matches the female pattern
        if word == "f" or re.search(female_pattern, word):
            return "female"

        # Check if the word directly matches 'm' or matches the male pattern
        if word == "m" or re.search(male_pattern, word):
            return "male"

        # Calculate fuzzy matching ratios between the word and 'male' and 'female'
        ratio1 = fuzz.ratio(word, "male")
        ratio2 = fuzz.ratio(word, "female")

        # Determine the gender based on the fuzzy matching ratios
        if ratio1 >= 70:
            res = "male"
        elif ratio2 >= 70:
            res = "female"
        else:
            res = None

        return res

    def process_crm_data(self, crm_s3_path):
        """
        Processes CRM data from an S3 bucket.

        Args:
            crm_s3_path (str): The S3 path of the CRM data file.

        Returns:
            DataFrame or None: The processed CRM data as a DataFrame if successful, None otherwise.
        """

        # Checking file exits or not in s3 bucket
        bucket_name = "telecom-data-source"
        key = "crm1.csv"

        file_flag = self.check_file_exists_in_S3(bucket_name=bucket_name, key=key)

        if not file_flag:
            # If the file doesn't exist, print an error message and return None
            print("Some error occured while getting data from s3 bucket")
            return None

        # Creating user define function to transform gender column.
        fuzzy_ratio_udf = self.spark.udf.register("fuzzy_ratio", self.fuzzy_ratio)

        #  Read CRM data from the provided s3 file
        crm_df = self.spark.read.option('header','true').option('inferSchema','true').format("csv").load(crm_s3_path)

        # # Clean and normalize 'gender' column values and replacing null values in year_of_birth with not available
        crm_df = crm_df.withColumn("gender", fuzzy_ratio_udf(col("gender")))\
                       .withColumn("year_of_birth", when(col("year_of_birth").isNull(), 'Not Available').otherwise(col("year_of_birth")))

        # Fill the last non-null value in the 'gender' column for each 'msisdn'
        window_spec = Window.partitionBy("msisdn").orderBy(col("year_of_birth"))
        crm_df = crm_df.withColumn("row_number1", row_number().over(window_spec))

        window_spec = Window.partitionBy("msisdn").orderBy(col("row_number1")).rowsBetween(Window.unboundedPreceding, Window.currentRow)
        crm_df = crm_df.withColumn('gender', last('gender', ignorenulls=True).over(window_spec))

        window_spec = Window.partitionBy("msisdn").orderBy(col("row_number1").desc()).rowsBetween(Window.unboundedPreceding, Window.currentRow)
        crm_df = crm_df.withColumn('gender', last('gender', ignorenulls=True).over(window_spec))
        # Filling all null with Not Available in gender column
        crm_df = crm_df.drop("row_number1")
        crm_df = crm_df.withColumn("gender", when(col("gender").isNull(), "Not Available").otherwise(col("gender")))

        # Create an outer window partitioned by "msisdn"
        outer_window = Window.partitionBy("msisdn")

        # Create a window partitioned by multiple columns including "msisdn", "year_of_birth", "mobile_type", and "gender"
        window = outer_window.partitionBy("msisdn", "year_of_birth", "mobile_type", "gender")

        # Drop duplicates
        crm_df = crm_df.dropDuplicates()

        # Adding required columns to the crm_df
        crm_df = crm_df.withColumn("outer_window", count("*").over(outer_window))\
                        .withColumn("no_of_deactive", count(when(col("system_status").isin("DEACTIVE"), True)).over(outer_window)) \
                        .withColumn("count_except_deactive", count(when(col("system_status").isin("ACTIVE", "SUSPEND", "IDLE"), True)).over(window))\
                        .withColumn("total_active", count(when((col("system_status") == "ACTIVE"), True)).over(outer_window))\
                        .withColumn("count_active", count(when(col("system_status") == 'ACTIVE', True)).over(window)) \
                        .withColumn("count_idle", count(when(col("system_status").isin("IDLE"), True)).over(window))\
                        .withColumn("total_idle", count(when((col("total_active") == 0) & (col("system_status") == "IDLE"), True)).over(outer_window))

        # Replacing the count_except_deactive columm value if it satisfy the given condition
        crm_df = crm_df.withColumn("count_except_deactive", when(col("outer_window") == col("count_except_deactive"), 1).otherwise(col("count_except_deactive")))

        # Replacing the count_except_deactive columm value if it satisfy the given conditions
        crm_df = crm_df.withColumn("count_except_deactive", when(((col("count_active") == 0) & (col("count_except_deactive") == 1) & (col("total_active") >= 1) & (col("no_of_deactive") == 0)), -1).
                                                when(((col("count_active") == 0) & (col("count_except_deactive") == 1) & (col("total_active") == 0) & (col("total_idle") >= 1) & (col("count_idle") == 0)), -1).
                                                otherwise(col("count_except_deactive")))\
                        .withColumn("taken", count(when((col("count_except_deactive") == 1) & (col("system_status") == "ACTIVE"), 1)).over(outer_window))

        # Filtering the based on count_except_deactive column value
        crm_df = crm_df.filter(~(col("count_except_deactive") == -1))

        #  # Replacing the count_except_deactive columm value if it satisfy the given conditions
        crm_df = crm_df.withColumn("count_except_deactive", when((col("total_active") >= 1) & (col("count_active") >= 1) & (col("taken") == 0), 1)
                                                            .when(((col("total_active") == 0) & (col("total_idle") >= 1) & (col("system_status") == "IDLE") & (col("count_idle") >= 1)), 1)
                                                            .when(((col("total_active") == 0) & (col("total_idle") == 0) & (col("system_status") == "SUSPEND") & (col("count_idle") == 0)), 1)
                                                            .otherwise(col("count_except_deactive")))
        # Filtering the records based on conditions
        crm_df = crm_df.filter(((col("system_status") == "DEACTIVE") & (col("no_of_deactive") != 0)) |
                                            ((col("count_except_deactive") == 1) & (col("system_status") == "ACTIVE") & (col("no_of_deactive") == 0)) |
                                            ((~(col("system_status") == "ACTIVE") & (col("count_active") == 0)  & (col("count_except_deactive") == 1) & (col("system_status") == "IDLE") & (col("no_of_deactive") == 0 ) &(col("total_active") == 0))) |
                                            (((col("count_except_deactive") == 1) & (col("count_idle") == 0) & (col("count_active") == 0 ) & (col("system_status") == "SUSPEND") & (col("no_of_deactive") == 0 ))) |
                                            (((col("total_active") >= col("count_active")) & (col("system_status") == "ACTIVE") & (col("count_except_deactive") == 1) & (col("no_of_deactive") == 0) & (col("taken") == 0))) |
                                            ((col("count_active") >= 1) & (col("system_status") == 'ACTIVE') & (col("no_of_deactive") == 0) & (col("taken") == 0)))

        # Drop the unnnessary columns
        crm_df= crm_df.drop("outer_window", "no_of_deactive", "count_except_deactive", "count_active", "count_idle", "total_idle", "total_active", "taken")

        # Filter duplicate 'msisdn' entries based on 'system_status'
        window_spec = Window.partitionBy("msisdn").orderBy(col("system_status").asc())
        crm_df = crm_df.withColumn("row_number", row_number().over(window_spec))
        crm_df = crm_df.filter(col("row_number") == 1).drop("row_number")

        # Write the processed CRM data to MongoDB
        self.mongo_writer.write_to_Mongodb(crm_df, 'mongodb://127.0.0.1/ProjectDB.crm1')

        # Writing to local
        # crm_df.coalesce(1).to_csv('results/crm_validate.csv')
        return crm_df

    def process_device_data(self, device_s3_path):
        """
        Processes device data from an S3 bucket.

        Args:
            device_s3_path (str): The S3 path of the device data file.

        Returns:
            DataFrame or None: The processed device data as a DataFrame if successful, None otherwise.
        """

        # Checking if the file exists in the S3 bucket
        bucket_name = "telecom-data-source"
        key = "device1.csv"
        file_flag = self.check_file_exists_in_S3(bucket_name=bucket_name, key=key)
        if not file_flag:
            print("Some error occurred while getting data from the S3 bucket")
            return None

        # Read device data from CSV
        device_df = self.spark.read.option("header", "true").option("inferSchema", "true").format("csv").load(device_s3_path)

        # Finding the number of nulls in each row
        device_df = device_df.withColumn("no_of_nulls", sum([col(c).isNull().cast('int') for c in device_df.columns]))

        # Remove duplicate 'msisdn' entries based on 'imei_tac' and select the record with fewer null values
        window_spec = Window.partitionBy("msisdn").orderBy('no_of_nulls')
        device_df = device_df.withColumn("row_number", row_number().over(window_spec))
        device_df = device_df.filter(col('row_number') == 1)
        device_df = device_df.drop("row_number").drop("no_of_nulls")

        # Fill missing values in the dataframe with "Not Available"
        device_df = device_df.fillna('Not Available')

        # Write the processed device data to MongoDB
        self.mongo_writer.write_to_Mongodb(device_df, 'mongodb://127.0.0.1/ProjectDB.device1')

        # Writing to local
        # device_df.coalesce(1).to_csv('results/device_validate.csv')

        return device_df

def databaseValidation(db_name, collection_name):
    """
    Validates the existence of a MongoDB database and collection, and checks if a document exists in the collection.

    Args:
        db_name (str): The name of the MongoDB database.
        collection_name (str): The name of the collection within the database.

    Returns:
        int: 1 if the database is validated, 0 otherwise.
    """

    # Create a MongoClient instance
    mongo_client = pymongo.MongoClient('mongodb://127.0.0.1:27017')

    try:
        # Get the list of database names from the MongoClient instance
        db_list = mongo_client.list_database_names()

        # Check if the specified database exists
        if db_name in db_list:
            # Get the specified database
            db = mongo_client.get_database(db_name)

            # Get the list of collection names within the database
            collections_list = db.list_collection_names()

            # Check if the specified collection exists
            if collection_name in collections_list:
                # Get the specified collection
                collection = db.get_collection(collection_name)

                # Find one document in the collection
                document = collection.find_one()

                # Check if a document exists in the collection
                if document is not None:
                    print("Database is validated")
                    return 1

    except pymongo.errors.PyMongoError as e:
        # Handle specific exceptions raised by PyMongo
        print(f"An error occurred during database validation: {str(e)}")

    return 0

if __name__ == '__main__':
    # Creating an object for initilizing spark
    processor = DataProcessor()

    # Validating the CRM database and processing CRM data if validation fails
    if databaseValidation('ProjectDB', "crm1") == 0:
        crm_df = processor.process_crm_data(CRM_PATH)

    # Validating the CRM database and processing device data if validation fails
    if databaseValidation('ProjectDB', "device1") == 0:
        device_df = processor.process_device_data(DEVICE_PATH)


import os
import sys
import time
import glob
import numpy as np
import torch
import utils
import logging
import argparse
import torch.nn as nn
import torch.utils
import torchvision.datasets as dset
import torch.backends.cudnn as cudnn

from dataset.imagenet_scripts import imagenet_data
from torch.autograd import Variable
import genotypes

from tools.lr_scheduler import get_lr_scheduler
from run_apis.trainer import Trainer
from tools.utils import cross_entropy_with_label_smoothing
from model import NetworkImageNet as Network
from configs.imagenet_train_cfg import cfg as config
from tools.utils import count_parameters_in_MB
from tools.multadds_count import comp_multadds

parser = argparse.ArgumentParser("cifar")
parser.add_argument('--gpus', type=str, default='0,1,2,3,4,5,6,7')  # '11,12,13,14'
parser.add_argument('--arch', type=str, default='RelativeNAS', help='which architecture to use')
parser.add_argument('--report_freq', type=float, default=50, help='report frequency')
parser.add_argument('--data_path', type=str, default='/raid/huangsh/imagenet/ILSVRC2012/lmdb')
# model setting
parser.add_argument('--init_channels', type=int, default=46, help='num of init channels')
parser.add_argument('--layers', type=int, default=14, help='total number of layers')
parser.add_argument('--drop_path_prob', type=float, default=0, help='drop path probability')
parser.add_argument('--cutout', action='store_true', default=False, help='use cutout')
parser.add_argument('--cutout_length', type=int, default=16, help='cutout length')
parser.add_argument('--seed', type=int, default=0, help='random seed')
args = parser.parse_args()

os.environ['CUDA_VISIBLE_DEVICES'] = '{}'.format(args.gpus)
args.save = '../exp_dirs/eval-imagenet-arch{}-l{}-c{}-lr{}'.format(args.arch, args.layers, args.init_channels,
                                                                   config.optim.init_lr)
utils.create_exp_dir(args.save)

log_format = '%(asctime)s %(message)s'
logging.basicConfig(stream=sys.stdout, level=logging.INFO, format=log_format, datefmt='%m/%d %I:%M:%S %p')
fh = logging.FileHandler(os.path.join(args.save, 'log.txt'))
fh.setFormatter(logging.Formatter(log_format))
logging.getLogger().addHandler(fh)
NUM_CLASSES = 1000

def adjust_lr(optimizer, epochs, learning_rate, epoch):
    # Smaller slope for the last 5 epochs because lr * 1/250 is relatively large
    if epochs - epoch > 5:
        lr = learning_rate * (epochs - 5 - epoch) / (epochs - 5)
    else:
        lr = learning_rate * (epochs - epoch) / ((epochs - 5) * 5)
    for param_group in optimizer.param_groups:
        param_group['lr'] = lr
    return optimizer, lr

class CrossEntropyLabelSmooth(nn.Module):
    def __init__(self, num_classes, epsilon):
        super(CrossEntropyLabelSmooth, self).__init__()
        self.num_classes = num_classes
        self.epsilon = epsilon
        self.logsoftmax = nn.LogSoftmax(dim=1)

    def forward(self, inputs, targets):
        log_probs = self.logsoftmax(inputs)
        targets = torch.zeros_like(log_probs).scatter_(1, targets.unsqueeze(1), 1)
        targets = (1 - self.epsilon) * targets + self.epsilon / self.num_classes
        loss = (-targets * log_probs).mean(0).sum()
        return loss

def main():
    np.random.seed(args.seed)
    cudnn.benchmark = True
    torch.manual_seed(args.seed)
    cudnn.enabled = True
    torch.cuda.manual_seed(args.seed)
    logging.info("args = %s", args)

    genotype = eval("genotypes.%s" % args.arch)
    model = Network(args.init_channels, NUM_CLASSES, args.layers, config.optim.auxiliary, genotype)

    start_epoch = 0
    model.eval()
    model.drop_path_prob = args.drop_path_prob * 0
    # compute the params as well as the multi-adds
    params = count_parameters_in_MB(model)
    logging.info("Params = %.2fMB" % params)
    mult_adds = comp_multadds(model, input_size=config.data.input_size)
    logging.info("Mult-Adds = %.2fMB" % mult_adds)

    model.train()
    if len(args.gpus) > 1:
        model = nn.DataParallel(model)
    model = model.cuda()
    if config.optim.label_smooth:
        criterion = CrossEntropyLabelSmooth(NUM_CLASSES, config.optim.smooth_alpha)
    else:
        criterion = nn.CrossEntropyLoss()
    criterion = criterion.cuda()

    optimizer = torch.optim.SGD(model.parameters(), config.optim.init_lr, momentum=config.optim.momentum,
                                weight_decay=config.optim.weight_decay)

    imagenet = imagenet_data.ImageNet12(trainFolder=os.path.join(args.data_path, 'train'),
                                        testFolder=os.path.join(args.data_path, 'val'),
                                        num_workers=config.data.num_workers,
                                        type_of_data_augmentation=config.data.type_of_data_aug, data_config=config.data,
                                        size_images=config.data.input_size[1], scaled_size=config.data.scaled_size[1])
    train_queue, valid_queue = imagenet.getTrainTestLoader(config.data.batch_size)

    if config.optim.lr_schedule == 'cosine':
        scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(config.train_params.epochs))

    trainer = Trainer(train_queue, valid_queue, criterion, config, args.report_freq)
    best_epoch = [0, 0, 0]  # [epoch, acc_top1, acc_top5]
    lr = config.optim.init_lr
    for epoch in range(start_epoch, config.train_params.epochs):
        if config.optim.lr_schedule == 'cosine':
            scheduler.step()
            current_lr = scheduler.get_lr()[0]
        elif config.optim.lr_schedule == 'linear':  # with warmup initial
            optimizer, current_lr = adjust_lr(optimizer, config.train_params.epochs, lr, epoch)
        else:
            print('Wrong lr type, exit')
            sys.exit(1)
        if epoch < 5:  # Warmup epochs for 5
            current_lr = lr * (epoch + 1) / 5.0
            for param_group in optimizer.param_groups:
                param_group['lr'] = current_lr
            logging.info('Warming-up Epoch: %d, LR: %e', epoch, lr * (epoch + 1) / 5.0)

        logging.info('Epoch: %d lr %e', epoch, current_lr)
        if len(args.gpus) > 1:
            model.module.drop_path_prob = args.drop_path_prob * epoch / config.train_params.epochs
        else:
            model.drop_path_prob = args.drop_path_prob * epoch / config.train_params.epochs
        train_acc_top1, train_acc_top5, train_obj, batch_time, data_time = trainer.train(model, optimizer, epoch)
        with torch.no_grad():
            val_acc_top1, val_acc_top5, batch_time, data_time = trainer.infer(model, epoch)
        if val_acc_top1 > best_epoch[1]:
            best_epoch = [epoch, val_acc_top1, val_acc_top5]
            if epoch >= 0:  # 120
                utils.save_checkpoint({
                    'epoch': epoch + 1,
                    'state_dict': model.module.state_dict(),
                    'best_acc_top1': val_acc_top1,
                    'optimizer': optimizer.state_dict(),
                }, save_path=args.save, epoch=epoch, is_best=True)
                if len(args.gpus) > 1:
                    utils.save(model.module.state_dict(),
                               os.path.join(args.save, 'weights_{}_{}.pt'.format(epoch, val_acc_top1)))
                else:
                    utils.save(model.state_dict(),
                               os.path.join(args.save, 'weights_{}_{}.pt'.format(epoch, val_acc_top1)))

        logging.info('BEST EPOCH %d  val_top1 %.2f val_top5 %.2f', best_epoch[0], best_epoch[1], best_epoch[2])
        logging.info('epoch: {} \t train_acc_top1: {:.4f} \t train_loss: {:.4f} \t val_acc_top1: {:.4f}'.format(epoch,
                                                                                train_acc_top1,
                                                                                train_obj,
                                                                                val_acc_top1))

    logging.info("Params = %.2fMB" % params)
    logging.info("Mult-Adds = %.2fMB" % mult_adds)

if __name__ == '__main__':
    main()


import logging
import os
import time
from pathlib import Path
from typing import Dict
from typing import List
from typing import Optional

from ruamel.yaml import YAMLError

from semgrep.constants import DEFAULT_CONFIG_FILE
from semgrep.constants import DEFAULT_CONFIG_FOLDER
from semgrep.constants import DEFAULT_SEMGREP_CONFIG_NAME
from semgrep.constants import ID_KEY
from semgrep.constants import PLEASE_FILE_ISSUE_TEXT
from semgrep.constants import RULES_KEY
from semgrep.constants import SEMGREP_USER_AGENT
from semgrep.constants import YML_EXTENSIONS
from semgrep.error import SemgrepError
from semgrep.error import UNPARSEABLE_YAML_EXIT_CODE
from semgrep.rule_lang import parse_yaml_preserve_spans
from semgrep.rule_lang import Span
from semgrep.rule_lang import YamlTree
from semgrep.util import is_url

logger = logging.getLogger(__name__)

IN_DOCKER = "SEMGREP_IN_DOCKER" in os.environ
IN_GH_ACTION = "GITHUB_WORKSPACE" in os.environ

SRC_DIRECTORY = Path(os.environ.get("SEMGREP_SRC_DIRECTORY", Path("/") / "src"))
OLD_SRC_DIRECTORY = Path("/") / "home" / "repo"

TEMPLATE_YAML_URL = (
    "https://raw.githubusercontent.com/returntocorp/semgrep-rules/develop/template.yaml"
)

RULES_REGISTRY = {"r2c": "https://semgrep.dev/c/p/r2c"}
DEFAULT_REGISTRY_KEY = "r2c"

def manual_config(pattern: str, lang: str) -> Dict[str, YamlTree]:
    # TODO remove when using sgrep -e ... -l ... instead of this hacked config
    pattern_span = Span.from_string(pattern, filename="CLI Input")
    pattern_tree = YamlTree[str](value=pattern, span=pattern_span)
    error_span = parse_yaml_preserve_spans(
        f"Semgrep bug generating manual config {PLEASE_FILE_ISSUE_TEXT}", filename=None
    ).span
    return {
        "manual": YamlTree.wrap(
            {
                RULES_KEY: [
                    {
                        ID_KEY: "-",
                        "pattern": pattern_tree,
                        "message": pattern,
                        "languages": [lang],
                        "severity": "ERROR",
                    }
                ]
            },
            span=error_span,
        )
    }

def resolve_targets(targets: List[str]) -> List[Path]:
    base_path = get_base_path()
    return [
        Path(target) if Path(target).is_absolute() else base_path.joinpath(target)
        for target in targets
    ]

def adjust_for_docker() -> None:
    # change into this folder so that all paths are relative to it
    if IN_DOCKER and not IN_GH_ACTION:
        if OLD_SRC_DIRECTORY.exists():
            raise SemgrepError(
                f"Detected Docker environment using old code volume, please use '{SRC_DIRECTORY}' instead of '{OLD_SRC_DIRECTORY}'"
            )
        if not SRC_DIRECTORY.exists():
            raise SemgrepError(
                f"Detected Docker environment without a code volume, please include '-v \"${{PWD}}:{SRC_DIRECTORY}\"'"
            )
    if SRC_DIRECTORY.exists():
        os.chdir(SRC_DIRECTORY)

def get_base_path() -> Path:
    return Path(".")

def indent(msg: str) -> str:
    return "\n".join(["\t" + line for line in msg.splitlines()])

def parse_config_at_path(
    loc: Path, base_path: Optional[Path] = None
) -> Dict[str, YamlTree]:
    """
        Assumes file at loc exists
    """
    config_id = str(loc)
    if base_path:
        config_id = str(loc).replace(str(base_path), "")

    with loc.open() as f:
        return parse_config_string(config_id, f.read(), str(loc))

def parse_config_string(
    config_id: str, contents: str, filename: Optional[str]
) -> Dict[str, YamlTree]:
    try:
        data = parse_yaml_preserve_spans(contents, filename)
        return {config_id: data}
    except YAMLError as se:
        raise SemgrepError(
            f"Invalid YAML file {config_id}:\n{indent(str(se))}",
            code=UNPARSEABLE_YAML_EXIT_CODE,
        )

def parse_config_folder(loc: Path, relative: bool = False) -> Dict[str, YamlTree]:
    configs = {}
    for l in loc.rglob("*"):
        # Allow manually specified paths with ".", but don't auto-expand them
        if not _is_hidden_config(l.relative_to(loc)) and l.suffix in YML_EXTENSIONS:
            if l.is_file():
                configs.update(parse_config_at_path(l, loc if relative else None))
    return configs

def _is_hidden_config(loc: Path) -> bool:
    """
    Want to keep rules/.semgrep.yml but not path/.github/foo.yml
    Also want to keep src/.semgrep/bad_pattern.yml but not ./.pre-commit-config.yaml
    """
    return any(
        part != "."
        and part != ".."
        and part.startswith(".")
        and DEFAULT_SEMGREP_CONFIG_NAME not in part
        for part in loc.parts
    )

def load_config_from_local_path(location: Optional[str] = None,) -> Dict[str, YamlTree]:
    """
        Return config file(s) as dictionary object
    """
    base_path = get_base_path()
    if location is None:
        default_file = base_path.joinpath(DEFAULT_CONFIG_FILE)
        default_folder = base_path.joinpath(DEFAULT_CONFIG_FOLDER)
        if default_file.exists():
            return parse_config_at_path(default_file)
        elif default_folder.exists():
            return parse_config_folder(default_folder, relative=True)
        else:
            return {}
    else:
        loc = base_path.joinpath(location)
        if loc.exists():
            if loc.is_file():
                return parse_config_at_path(loc)
            elif loc.is_dir():
                return parse_config_folder(loc)
            else:
                raise SemgrepError(f"config location `{loc}` is not a file or folder!")
        else:
            addendum = ""
            if IN_DOCKER:
                addendum = " (since you are running in docker, you cannot specify arbitary paths on the host; they must be mounted into the container)"
            raise SemgrepError(
                f"unable to find a config; path `{loc}` does not exist{addendum}"
            )

def nice_semgrep_url(url: str) -> str:
    """
    Alters semgrep.dev urls to let user
    click through to the nice display page instead
    of raw YAML.
    Replaces '/c/' in semgrep urls with '/'.
    """
    import urllib

    parsed = urllib.parse.urlparse(url)
    if "semgrep.dev" in parsed.netloc and parsed.path.startswith("/c"):
        return url.replace("/c/", "/")
    return url

def download_config(config_url: str) -> Dict[str, YamlTree]:
    import requests  # here for faster startup times

    DOWNLOADING_MESSAGE = f"downloading config..."
    logger.debug(f"trying to download from {config_url}")
    logger.info(
        f"using config from {nice_semgrep_url(config_url)}. Visit https://semgrep.dev/registry to see all public rules."
    )
    logger.info(DOWNLOADING_MESSAGE)
    headers = {"User-Agent": SEMGREP_USER_AGENT}

    try:
        r = requests.get(config_url, stream=True, headers=headers, timeout=10)
        if r.status_code == requests.codes.ok:
            content_type = r.headers.get("Content-Type")
            yaml_types = [
                "text/plain",
                "application/x-yaml",
                "text/x-yaml",
                "text/yaml",
                "text/vnd.yaml",
            ]
            if content_type and any((ct in content_type for ct in yaml_types)):
                return parse_config_string(
                    "remote-url",
                    r.content.decode("utf-8"),
                    filename=f"{config_url[:20]}...",
                )
            else:
                raise SemgrepError(
                    f"unknown content-type: {content_type} returned by config url: {config_url}. Can not parse"
                )
        else:
            raise SemgrepError(
                f"bad status code: {r.status_code} returned by config url: {config_url}"
            )
    except Exception as e:
        raise SemgrepError(f"Failed to download config from {config_url}: {str(e)}")

    return None

def resolve_config(config_str: Optional[str]) -> Dict[str, YamlTree]:
    """ resolves if config arg is a registry entry, a url, or a file, folder, or loads from defaults if None"""
    start_t = time.time()
    if config_str is None:
        config = load_config_from_local_path()
    elif config_str in RULES_REGISTRY:
        config = download_config(RULES_REGISTRY[config_str])
    elif is_url(config_str):
        config = download_config(config_str)
    else:
        config = load_config_from_local_path(config_str)
    if config:
        logger.debug(f"loaded {len(config)} configs in {time.time() - start_t}")
    return config

def generate_config() -> None:
    import requests  # here for faster startup times

    # defensive coding
    if Path(DEFAULT_CONFIG_FILE).exists():
        raise SemgrepError(
            f"{DEFAULT_CONFIG_FILE} already exists. Please remove and try again"
        )
    try:
        r = requests.get(TEMPLATE_YAML_URL, timeout=10)
        r.raise_for_status()
        template_str = r.text
    except Exception as e:
        logger.debug(str(e))
        logger.warning(
            f"There was a problem downloading the latest template config. Using fallback template"
        )
        template_str = """rules:
  - id: eqeq-is-bad
    pattern: $X == $X
    message: "$X == $X is a useless equality check"
    languages: [python]
    severity: ERROR"""
    try:
        with open(DEFAULT_CONFIG_FILE, "w") as template:
            template.write(template_str)
            logger.info(
                f"Template config successfully written to {DEFAULT_CONFIG_FILE}"
            )
    except Exception as e:
        raise SemgrepError(str(e))


from setuptools import setup

with open("README.md", "r") as f:
    description = f.read()

setup(
    name='QtAnim',
    version='0.0.1',
    description='Easy animation for PyQt!',
    py_modules=['QtAnim'],
    package_dir={'': 'src'},
    long_description=description,
    long_description_content_type='text/markdown',
    extras_require = {
        "dev" : [
            "pytest>=3.7",
        ],
    },
)


from dataLoad import create_data
import numpy as np
import sys
sys.path.append('..')
from config import Config
import torch

class DataIterator(object):
    def __init__(self, config, data_file, is_test=False):
        self.batch_size = config.batch_size
        self.seq_length = config.sequence_length
        self.tokenizer = config.tokenizer
        self.device = config.device
        # 数据的操作
        self.data = create_data(data_file)  # 样本数据对象数组
        self.num_records = len(self.data)  # 数据的个数
        self.idx = 0  # 数据索引
        self.all_idx = list(range(self.num_records))  # 全体数据索引
        self.is_test = is_test

        self.data_type = data_file.split('/')[-1].split('.')[0]
        # 训练模式下数据乱序
        if not self.is_test:
            np.random.shuffle(self.all_idx)

        print(self.data_type,"样本个数：", self.num_records)

    def convert_single_example(self, example_idx):
        '''处理单个数据样本'''
        textA_list = list(self.data[example_idx].textA)
        textB_list = list(self.data[example_idx].textB)
        label = self.data[example_idx].label
        # 将文本转为词典索引
        tokens_A = []
        tokens_B = []
        for index, token in enumerate(textA_list):
            char_list = self.tokenizer.tokenize(token.lower())
            if char_list:
                tokens_A.append(char_list[0])
            else:
                # 解析不到的特殊字符用 §替代
                tokens_A.append("§")
        for index, token in enumerate(textB_list):
            char_list = self.tokenizer.tokenize(token.lower())
            if char_list:
                tokens_B.append(char_list[0])
            else:
                # 解析不到的特殊字符用 §替代
                tokens_B.append("§")
        # 加入修饰符
        tokens = ["[CLS]"] + tokens_A + ["[SEP]"] + tokens_B + ["[SEP]"]
        # 将序列文字转为字典索引
        input_ids = self.tokenizer.convert_tokens_to_ids(tokens)

        # 初始化单个样本的各个状态向量
        input_ids_cpu = np.zeros((1, self.seq_length), dtype=np.int64)
        attention_mask_cpu = np.zeros((1, self.seq_length), dtype=np.int64)
        token_type_ids_cpu = np.zeros((1,self.seq_length), dtype=np.int64)
        label_cpu = np.zeros((1,), dtype=np.int64)
        tokens_cpu = np.array(['*NULL*'] * self.seq_length)
        tokens_cpu = np.expand_dims(tokens_cpu, axis=0)

        # 复制单个样本的状态向量
        # 个别样本长度超出作截断
        input_ids_cpu[0, :len(input_ids)] = input_ids if len(input_ids) <= self.seq_length else input_ids[:self.seq_length]
        attention_mask_cpu[0, :len(input_ids)] = [1] * len(input_ids) if len(input_ids) <= self.seq_length else [1] * self.seq_length

        if len(input_ids) <= self.seq_length:
            token_type_ids_cpu[0, (len(tokens_A) + 2): len(input_ids)] = 1
            tokens_cpu[0, :len(tokens)] = tokens
        else:
            token_type_ids_cpu[0, (len(tokens_A) + 2):] = 1
            tokens_cpu[0,:] = tokens[:self.seq_length]

        label_cpu[0] = label

        # input_ids_cpu[0, :len(input_ids)] = input_ids
        # attention_mask_cpu[0, :len(input_ids)] = [1] * len(input_ids)
        # token_type_ids_cpu[0, (len(tokens_A) + 2): len(input_ids)] = 1
        # label_cpu[0] = label
        # tokens_cpu[0, :len(tokens)] = tokens

        return input_ids_cpu, attention_mask_cpu, token_type_ids_cpu, label_cpu, tokens_cpu

    def __iter__(self):
        return self

    def __next__(self):
        # 初始化存储一个batch状态向量的变量
        input_ids_cpu, attention_mask_cpu, token_type_ids_cpu, label_cpu, tokens_cpu = [], [], [], [], []
        example_count = 0

        # 迭代数量超出总数量时
        if self.idx >= self.num_records:
            self.idx = 0
            if not self.is_test:
                np.random.shuffle(self.all_idx)
            raise StopIteration

        # 返回一个batch的数据
        while example_count < self.batch_size:
            # 依次读取乱序后的数据样本
            idx = self.all_idx[self.idx]
            res = self.convert_single_example(idx)

            single_input_ids, single_attention_mask, single_token_type_ids, single_label, single_tokens = res

            # 把样本逐个堆叠成一个batch的数据
            if example_count == 0:
                input_ids_cpu, attention_mask_cpu, token_type_ids_cpu, label_cpu, tokens_cpu = \
                    single_input_ids, single_attention_mask, single_token_type_ids, single_label, single_tokens
            else:
                input_ids_cpu = np.concatenate((input_ids_cpu, single_input_ids), axis=0)
                attention_mask_cpu = np.concatenate((attention_mask_cpu, single_attention_mask), axis=0)
                token_type_ids_cpu = np.concatenate((token_type_ids_cpu, single_token_type_ids), axis=0)
                label_cpu = np.concatenate((label_cpu, single_label), axis=0)
                tokens_cpu = np.concatenate((tokens_cpu, single_tokens), axis=0)

            example_count += 1
            self.idx += 1
            if self.idx >= self.num_records:
                break

        input_ids = torch.from_numpy(input_ids_cpu).to(self.device)
        attention_mask = torch.from_numpy(attention_mask_cpu).to(self.device)
        token_type_ids = torch.from_numpy(token_type_ids_cpu).to(self.device)
        labels = torch.from_numpy(label_cpu).to(self.device)

        return input_ids, attention_mask, token_type_ids, labels, tokens_cpu

    def __len__(self):
        if self.num_records % self.batch_size == 0:
            return self.num_records // self.batch_size
        else:
            return self.num_records // self.batch_size + 1

if __name__ == '__main__':
    config = Config()
    test_iter = DataIterator(config, config.source_data_dir + 'test.csv', is_test=True)
    print(len(test_iter))
    for input_ids, attention_mask, token_type_ids, labels, tokens_list in test_iter:
        print(input_ids)
        print(attention_mask)
        print(token_type_ids)
        print(labels)
        print(tokens_list)


"""
"""

import phantom.rules as phantom
import json
from datetime import datetime, timedelta

##############################
# Start - Global Code Block

import requests

# End - Global Code block
##############################

def on_start(container):
    phantom.debug('on_start() called')

    # call 'promote_to_case_1' block
    promote_to_case_1(container=container)

    return

def promote_to_case_1(action=None, success=None, container=None, results=None, handle=None, filtered_artifacts=None, filtered_results=None):
    phantom.debug('promote_to_case_1() called')

    phantom.promote(container=container, template="responses")
    get_case_note_count(container=container)

    return

def get_case_note_count(action=None, success=None, container=None, results=None, handle=None, filtered_artifacts=None, filtered_results=None):
    phantom.debug('no_op_1() called')

    # get the container id and phantom url to format for the request
    container_id = container.get('id', None)
    phantom_url = phantom.get_base_url()
    request_url = "{}/rest/container/{}/phases".format(str(phantom_url), str(container_id))

    # make the request
    r = requests.get(request_url, auth=("admin","password"), verify=False).json()

    # check to see if all notes fields are filled out
    notes_counter = 0
    for i in r["data"][0]["tasks"]:
        phantom.debug(i)
        if i["notes"]:
            notes_counter = notes_counter + 1

    # if all the fields are filled out prompt before emailing, if not sleep and check again
    if notes_counter != 3:
        no_op_2(container=container, handle=notes_counter)
    else:
        for i in r["data"][0]["tasks"]:
            raw = {}
            cef = {}
            cef['container_note'] = i["notes"][0]["content"]

            success, message, artifact_id = phantom.add_artifact(
                container=container, raw_data=raw, cef_data=cef, label='note',
                name='container note', severity='low',
                identifier=None,
                artifact_type='note')

        prompt_1(container=container)

    return

def send_email_1(action=None, success=None, container=None, results=None, handle=None, filtered_artifacts=None, filtered_results=None):
    phantom.debug('send_email_1() called')

    #phantom.debug('Action: {0} {1}'.format(action['name'], ('SUCCEEDED' if success else 'FAILED')))

    # collect data for 'send_email_1' call
    formatted_data_1 = phantom.get_format_data(name='format_1')

    parameters = []

    # build parameters list for 'send_email_1' call
    parameters.append({
        'body': formatted_data_1,
        'from': "",
        'attachments': "",
        'to': "niroy@splunk.com",
        'cc': "",
        'bcc': "",
        'headers': "",
        'subject': "Test Container Note",
    })

    phantom.act("send email", parameters=parameters, assets=['smtp'], name="send_email_1")

    return

def no_op_2(action=None, success=None, container=None, results=None, handle=None, filtered_artifacts=None, filtered_results=None):
    # handle passes count from the previous get_case_note_count

    phantom.debug('no_op_2() called')

    parameters = []

    # build parameters list for 'no_op_2' call
    parameters.append({
        'sleep_seconds': 10,
    })

    #
    if handle != 3:
        phantom.act("no op", parameters=parameters, assets=['local phantom'], callback=get_case_note_count, name="no_op_2", parent_action=action)
    else:
        prompt_1(container=container)
    return

def prompt_1(action=None, success=None, container=None, results=None, handle=None, filtered_artifacts=None, filtered_results=None):
    phantom.debug('prompt_1() called')

    # set user and message variables for phantom.prompt call
    user = "admin"
    message = """is there enough information to send out"""

    # response options
    options = {
        "type": "list",
        "choices": [
            "Yes",
            "No",
        ]
    }

    phantom.prompt(container=container, user=user, message=message, respond_in_mins=30, name="prompt_1", options=options, callback=decision_2)

    return

def decision_2(action=None, success=None, container=None, results=None, handle=None, filtered_artifacts=None, filtered_results=None):
    phantom.debug('decision_2() called')

    # check for 'if' condition 1
    matched_artifacts_1, matched_results_1 = phantom.condition(
        container=container,
        action_results=results,
        conditions=[
            ["prompt_1:action_result.summary.response", "==", "Yes"],
        ])

    # call connected blocks if condition 1 matched
    if matched_artifacts_1 or matched_results_1:
        format_1(action=action, success=success, container=container, results=results, handle=handle)
        return

    # call connected blocks for 'else' condition 2

    return

def format_1(action=None, success=None, container=None, results=None, handle=None, filtered_artifacts=None, filtered_results=None):
    phantom.debug('format_1() called')

    template = """{0}"""

    # parameter list for template variable replacement
    parameters = [
        "artifact:*.cef.container_note",
    ]

    phantom.format(container=container, template=template, parameters=parameters, name="format_1")

    send_email_1(container=container)

    return

def on_finish(container, summary):
    phantom.debug('on_finish() called')
    # This function is called after all actions are completed.
    # summary of all the action and/or all detals of actions
    # can be collected here.

    # summary_json = phantom.get_summary()
    # if 'result' in summary_json:
        # for action_result in summary_json['result']:
            # if 'action_run_id' in action_result:
                # action_results = phantom.get_action_results(action_run_id=action_result['action_run_id'], result_data=False, flatten=False)
                # phantom.debug(action_results)

    return


# [X] Load in questions - json
# [ ] iterate our questions
# [ ] Check for invalid inputs
# [ ] Provide way to quit
# [ ] final score

from Question import collectQuestions
from random import shuffle
from time import sleep

def intput(message:str, errorMessage:str='Invalid value, please give a valid integer.'):
	# infinite loop to try and get input until it's valid
	while True:
		userInput = input(message)

		# Checking if valid input received
		if userInput.isdigit():
			break

		# If invalid input, print message and repeat
		print(errorMessage)
		sleep(.5)

	return int(userInput)

# Grabbing our list of questions and randomizing order
questions = collectQuestions()
shuffle(questions)

# Tracking variables
answersGiven = 0
answersCorrect = 0

print('''
████████╗██████╗░██╗██╗░░░██╗██╗░█████╗░
╚══██╔══╝██╔══██╗██║██║░░░██║██║██╔══██╗
░░░██║░░░██████╔╝██║╚██╗░██╔╝██║███████║
░░░██║░░░██╔══██╗██║░╚████╔╝░██║██╔══██║
░░░██║░░░██║░░██║██║░░╚██╔╝░░██║██║░░██║
░░░╚═╝░░░╚═╝░░╚═╝╚═╝░░░╚═╝░░░╚═╝╚═╝░░╚═╝

Enter '0' at any time to quit.

Enter the maximum number of questions you would like to answer.''')
numberQuestions = intput('  : ')

# Iterating our questions
for index, question in enumerate(questions):
	# Checking if our max question count has been reached
	if index >= numberQuestions:
		break

	# Prompt user
	print(question)
	answer = intput(
		'  Response: ',
		'    ERROR: Invalid response, please enter the index of your guess.'
	)

	# Checking if user requested exit
	if answer <= 0:
		break # Exit for each question

	# Checking if user is correct
	correct = question.checkAnswer(answer-1)

	# Adding to our total
	answersGiven += 1
	if correct:
		answersCorrect += 1

	# User feedback
	if correct:
		print('That\'s correct, great job!')
	else:
		print('That\'s wrong, sorry!')
	sleep(1)
	print('\n\n')

# Print summary of trivia
print(f'You guessed {answersCorrect} questions correct out of {answersGiven} questions answered!')
accuracy = answersCorrect / answersGiven * 100
print(f'That\'s an accurracy of {accuracy:.0f}%')


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
@time    : 2023/1/10
@file    : chart json 前后结果比较
"""

import re
import json
import os
import sys
from collections import OrderedDict

def compare(file1, file2):
    with open(file1, 'r') as json1:
        data1 = json.load(json1)

    with open(file2, 'r') as json2:
        data2 = json.load(json2)

    for key, value in data1.items():
        pres = key
        if key == "dataset":
            pass
        else:
            if key in data2:
                compare_one(pres, data1[key], data2[key])
            else:
                print("{} lacks key".format(key))

def compare_one(pres, data1, data2):
    if type(data1) != type(data2):
        print("pres type error {}".format(pres))
    elif type(data1) == dict:
        for key, value in data1.items():
            pres += "." + key
            if key not in data2:
                print("{} lack in old".format(pres))
            else:
                compare_one(pres, data1[key], data2[key])
    elif type(data1) == list:
        for k, value in enumerate(data1):
            pres += "." + str(k)
            if k in data2:
                compare_one(pres, data1[k], data2[k])
            else:
                print("{} lacks key".format(pres))
    else:
        if data1 != data2:
            print("{}  not equal {} {}".format(pres, data1, data2))

if __name__ == '__main__':
    file1 = sys.argv[1]
    file2 = sys.argv[2]
    compare(file1, file2)


import plotly.graph_objs as go
import numpy as np

def get_survival_plotly_graphs(graph_df, x, y, y_upper, y_lower, color, show_lines_only):
    survival_line = go.Scatter(
        name=f'Churn Risk {graph_df.name or "Baseline"}',
        x=x,
        y=y,
        mode='lines',
        line=dict(color=color),
        line_shape="hv",
    )
    if show_lines_only:
        return [survival_line]

    confidence_upper = go.Scatter(
        name='Upper Confidence',
        x=x,
        y=y_upper,
        mode='lines',
        marker=dict(color="#444"),
        line=dict(width=0),
        showlegend=False,
        line_shape="hv",
    )
    confidence_lower = go.Scatter(
        name='Lower Confidence',
        x=x,
        y=y_lower,
        marker=dict(color="#444"),
        line=dict(width=0),
        mode='lines',
        fillcolor='rgba(68, 68, 68, 0.3)',
        fill='tonexty',
        showlegend=False,
        line_shape="hv",
    )
    active_customers = go.Scatter(
        name="Active Customer Volume",
        mode="markers",
        x=x,
        y=y,
        hovertemplate="%{text}",
        text=graph_df["active_at_risk"],
        marker=dict(color="rgba(31, 119, 180, 1)", line=dict(width=0), size=(graph_df["active_at_risk"] / np.max(graph_df["active_at_risk"])) * 19 + 1),
    )
    censored_customers = go.Scatter(
        name="Censored Customers",
        mode="markers",
        x=x,
        y=y,
        hovertemplate="%{text}",
        text=graph_df["censored"],
        marker=dict(color="rgba(124, 144, 163, 1)", line=dict(width=0), size=(graph_df["censored"] / np.max(graph_df["censored"])) * 19 + 1),
    )
    churned_customers = go.Scatter(
        name="Churned Customers",
        mode="markers",
        x=x,
        y=y,
        hovertemplate="%{text}",
        text=graph_df["observed"],
        marker=dict(color="rgba(227, 115, 111, 1)", line=dict(width=0), size=(graph_df["observed"] / np.max(graph_df["observed"])) * 19 + 1),
    )

    return [
        confidence_upper,
        confidence_lower,
        churned_customers,
        censored_customers,
        active_customers,
        survival_line,
    ]

def get_survival_plotly_fig(graph_dfs: list, survival_type, show_lines_only) -> go.Figure:
    colors = [
        "rgb(34, 124, 157)",
        "rgb(23, 195, 178)",
        "rgb(255, 203, 119)",
        "rgb(254, 109, 115)",
    ]

    all_traces = []
    for i, graph_df in enumerate(graph_dfs):
        if graph_df is None:
            continue
        x = graph_df["survival_duration"]
        y = (1 - graph_df["estimated_surviving_percent"])
        # These are reversed since y is reversed
        y_upper = (1 - graph_df["confidence_lower"])
        y_lower = (1 - graph_df["confidence_upper"])
        all_traces += get_survival_plotly_graphs(graph_df, x, y, y_upper, y_lower, colors[i%len(colors)], show_lines_only)

    fig = go.Figure(all_traces)

    fig.update_traces(hoverlabel={"namelength" :-1})

    n_subscriptions = graph_dfs[0]["all_at_risk"].iloc[0]
    fig.update_layout(title=f'Ran on {n_subscriptions} Subscriptions')
    fig.update_layout(xaxis_title=survival_type.title().replace("_", " "))
    fig.update_layout(yaxis_title='Churn Risk', yaxis_range=[-.05, 1.05], yaxis_tickformat='.1%')
    fig.update_layout(hovermode="x")

    return fig


import os, datetime
from shutil import copy2

replace_these_things = {  # use this dictionary to automatically replace things in your .tex to prep it for conversion to .docx; for example,
                r"\some_command_that_needs_to_be_commented_out": r"%\some_command_that_needs_to_be_commented_out"
                }

def unique(list1):
    unique_list = []
    for x in list1:
        if x not in unique_list:
            unique_list.append(x)
    return unique_list

def tabularize_eqns(fin, eqn_ids, cleanup=True):
    eqn_lookups = ["equation"]
    table_fill = [r"\begin{tabular}{cc}", r" & ", r" \end{tabular}"]
    tempfile = fin.replace(".tex", "_temp.tex")
    fout = fin.replace(".tex", "_temp_out.tex")
    copy2(src=fin, dst=tempfile)
    with open(tempfile, "r") as fp:
        lines = fp.readlines()
    eqn_starts = []
    eqn_ends = []
    eqn_replace = []
    for e in eqn_lookups:
        eqn_start = r"\begin{" + e + r"}"
        eqn_end = r"\end{" + e + r"}"
        starts = [i for i, n in enumerate(lines) if eqn_start in n]
        ends = [i+1 for i, n in enumerate(lines) if eqn_end in n]
        replace_flag = [False for i in range(len(starts))]
        for i in range(len(starts)):
            temp_eqn_lines = [j for j in range(starts[i], ends[i])]
            replace_eqn = False
            for id in eqn_ids.keys():
                label = r"\label" + id
                for j in temp_eqn_lines:
                    if label in lines[j]:
                        replace_eqn = True
            replace_flag[i] = replace_eqn
        eqn_starts = eqn_starts + starts
        eqn_ends = eqn_ends + ends
        eqn_replace = eqn_replace + replace_flag
    for i in range(len(eqn_starts)):
        if eqn_replace[i]:
            temp_eqn_lines = [j for j in range(eqn_starts[i], eqn_ends[i])]
            lines[temp_eqn_lines[0]] = lines[temp_eqn_lines[0]].replace(lines[temp_eqn_lines[0]], table_fill[0] + "\n" + lines[temp_eqn_lines[0]])
            for id in eqn_ids.keys():
                label = r"\label" + id
                temp_rep = False
                for j in temp_eqn_lines:
                    if label in lines[j]:
                        lines[j] = lines[j].replace(label, "")
                        temp_rep = True
                if temp_rep:
                    lines[temp_eqn_lines[-1]] = lines[temp_eqn_lines[-1]].replace(lines[temp_eqn_lines[-1]], lines[temp_eqn_lines[-1]] + table_fill[1] + label + "\n" + table_fill[2] + "\n")
    with open(fout, "wt") as fo:
        for line in lines:
            fo.write(line)
    if cleanup:
        os.remove(tempfile)
    return fout

def tex_label_ids(fin, id_flags=None, cleanup=True):
    label_flag = r"\label{"
    if id_flags is None:
        id_flags = ["eq:", "eqn:"]
    id_flags = [label_flag + e for e in id_flags]
    tempfile = fin.replace(".tex", "_temp.tex")
    copy2(src=fin, dst=tempfile)
    count = 0
    id_labels = []
    lookups = []
    id_labels2 = []
    ids = {}
    with open(tempfile) as fp:
        for line in fp:
            count += 1
            # print("Line{}: {}".format(count, line.strip()))
            for a in id_flags:
                if a in line:
                    id_labels = id_labels + [line]
                    lookups = lookups + [a]
    for i in range(len(id_labels)):
        tempe = id_labels[i].split(lookups[i])
        tempe = tempe[1].split(r"}")
        id = tempe[0]
        id_labels2 = id_labels2 + [lookups[i] + id + "}"]
    id_labels2_unique = unique(id_labels2)
    if len(id_labels2) != len(id_labels2_unique):
        print("WARNING: non-unique labels detected")
    id_count = 0
    for e in id_labels2_unique:
        id_count += 1
        id = e.split(r"\label")
        ids[id[1]] = str(id_count)
    print(ids)
    if cleanup:
        os.remove(tempfile)
    return ids

def tex_number_ref(fin, ids, label_prefix=None):
    tempfile = fin.replace(".tex", "_temp.tex")
    # count = 0
    if len(ids) > 0:
        with open(fin) as fi:
            with open(tempfile, "wt") as fo:
                for line in fi:
                    # count += 1
                    for id,n in ids.items():
                        if id in line:
                            # print(line)
                            if label_prefix is None:
                                line = line.replace(r"\label" + id, "(" + n + ")")
                            else:
                                line = line.replace(r"\label" + id, label_prefix + n)
                            line = line.replace(r"\ref" + id, n)
                            line = line.replace(r"\eqref" + id, "(" + n + ")")
                            # print(line)
                    fo.write(line)
    else:
        tempfile = fin
    return tempfile

def prep_eqns(fin, id_flags=None, cleanup=True):
    if id_flags is None:
        id_flags = ["eq:", "eqn:"]
    ids = tex_label_ids(fin, id_flags=id_flags, cleanup=cleanup)
    midfile = tabularize_eqns(fin, ids, cleanup=cleanup)
    newfile = tex_number_ref(midfile, ids)
    if cleanup:
        os.remove(midfile)
    return newfile

def prep_figs(fin, id_flags=None, cleanup=True, label_prefix="Figure "):
    if id_flags is None:
        id_flags = ["fig:"]
    ids = tex_label_ids(fin, id_flags=id_flags, cleanup=cleanup)
    newfile = tex_number_ref(fin, ids, label_prefix=label_prefix)
    return newfile

def pref_file(file=None, fileout=None):
    timestamp = str(datetime.datetime.now()).replace(" ", "_").replace(".", "_").replace(":", "-")
    tempfile = "temp_" + timestamp + ".tex"
    if fileout is None:
        fileout = tempfile
    copy2(file, tempfile)
    with open(tempfile, "r") as f:
        Lines = f.readlines()
    count = 0
    for line in Lines:
        for a in replace_these_things.keys():
            if a in line:
                print("Commenting out certain commands...")
                print("Replacing some code...")
                line = line.replace(a, replace_these_things[a])
        Lines[count] = line
        count += 1
    print("Writing temporary file...")
    with open(fileout, "w") as f:
        f.writelines(Lines)
    return fileout

def run_pandoc(file_in=None, file_out=None, refs=None, template=None, toc=True, header=None, ref_style=None):
    if file_out is None:
        file_out = file_in.replace(".tex", ".docx")
    if refs is None:
        refs_command = " --bibliography=refs.bib"
    else:
        refs_command = " --bibliography=" + refs
    if template is None:
        template_command = ""
    else:
        template_command = " --reference-doc=" + template
    if toc:
        table_of_contents = " --table-of-contents"
    else:
        table_of_contents = ""
    if header is None:
        header_command = ""
    else:
        header_command = " -H " + header
    if ref_style is None:  # default to chicago style
        ref_style_command = " --csl=./pandoc/chicago-author-date.csl"
    else:
        ref_style_command = " --csl=" + ref_style
    pandoc_command = "pandoc -s " + file_in + header_command + template_command + table_of_contents + " --standalone --filter pandoc-xnos" + refs_command + ref_style_command + " --citeproc -o " + file_out
    print("Running pandoc: ")
    print(pandoc_command)
    os.system(pandoc_command)
    return None

def tex2docx(filein=None, fileout=None, cleanup=False, refs=None, template=None, toc=True, header=None, ref_style=None):
    if fileout is None:
        fileout = "\"" + filein.replace(".tex", ".docx") + "\""
    if refs is None:
        refs = "refs.bib"
    tempfile1 = prep_eqns(filein, cleanup=cleanup)
    tempfile2 = prep_figs(tempfile1, cleanup=cleanup)
    tempfile3 = prep_figs(tempfile2, id_flags=["tab:"], cleanup=cleanup, label_prefix="Table ")
    tempfile4 = pref_file(file=tempfile3)
    run_pandoc(file_in=tempfile4, file_out=fileout, refs=refs, template=template, toc=toc, header=header, ref_style=ref_style)
    if cleanup:
        print("Removing temporary files...")
        tempfiles = [tempfile1, tempfile2, tempfile3, tempfile4]
        for f in tempfiles:
            if os.path.isfile(f):
                os.remove(f)
    return None

if __name__ == "__main__":
    # fin = "test.tex"
    # eids = tex_label_ids(fin=fin, id_flags=["eq:", "eqn:"])
    # fids = tex_label_ids(fin=fin, id_flags=["fig:"])
    # newfile = tabularize_eqns(fin, eqn_ids=eids)
    # newfile = prep_eqns(fin)
    # newfile2 = prep_figs(newfile)
    tex2docx(filein="Example.tex", refs="refs.bib", cleanup=True)