'''
Description
'''
from __future__ import division
from __future__ import print_function
from builtins import range
from builtins import object
from past.utils import old_div
__filename__ = "genepopfilemanager.py"
__date__ = "20160505"
__author__ = "Ted Cosart<ted.cosart@umontana.edu>"

import sys
import os
import random
from agestrucne.genepopindividualid import GenepopIndivIdVals
from agestrucne.genepopindividualid import GenepopIndividualId

COMMA_DELIMITED_LOCI_LIST_HAS_LEADING_SPACE=True
'''
2017_05_01.  Py3 compatiblity requires decoding
file lines read in from a file opened 'rb'.
This value is accessed dynamically in defs
__get_indiv_entry and getListIndividuals.
'''
SYSENCODING=sys.getdefaultencoding()

class GenepopFileManager( object ):

	'''
	wraps 2 dictionaries, one for header and loci byte addresses, the other for
			byte addresses of "pop" lines and individual addresses.  The header dict
			has one level, the pop dict  has 3 levels:
			outer key: for the pop dict, pop number: 1,2..., for header/loci simply the line number
			middle key: pop dict only,   item number: 0,1,2
				for the pop dictionsary the first "pop" line itself is the zeroth item
				and for pop 1,2,..., the entry "pop"
			innermost key:  item-line number, 0,1,2...
				this key needed because genepop allows multi line entries for
				a single individual
		value: byte address of the first byte of the line

		example, if (as most often) the individual entry takes up only one line,
		to find the entry for the 2nd indiv at pop 2, point the file pointer at
		the byte address given by self.__byte_addresses[2][2][1], and execute a readline().
		If this individual was on more than one line, say 2 lines, the entry would 8
		be given by concatenating the lines read at self.__byte_addresses[2][2][1]
		and self.__byte_addresses[2][2][2]

		Note that we aim to parse the genepop file in accordance with the format described
		in http://genepop.curtin.edu.au/help_input.html#Input as accessed most recently
		on 2016_10_03.
	'''

	'''
	These constants are added on 2016_10_03
	and will be incorporated into new code
	as reminders of the correct keys that
	fetches the dictionary that holds byte addresse(s)
	(plural only if the id + loci are entered
	on multiple lines) of the first indiv:
	'''
	KEY_FIRST_INDIV_NUMBER=1
	'''
	And for any indiv, the key to the first
	(usually the only) line that holds id + loci:
	'''
	KEY_FIRST_LINE_INDIV_ENTRY=1

	'''
	THe "pop" entry, signalling the start
	of a new population, is always the first
	key ( value = 0 ) in a dict of byte addresses
	for a given pop.
	'''
	KEY_POP_ENTRY=0

	def __init__( self, s_filename ):
		self.__filename=s_filename
		self.__loci_count=None

		self.__setup_addresses( s_filename )
		self.__init_subsamples()
		self.__get_loci_count()
		return
	#end __init__

	def __setup_addresses( self, s_filename ):
		i_currver=sys.version_info.major

		#for compat with python 3 (we need to cast
		#byte addresses as long in python 2, else very big files
		#have byte addresses that overrun py2's max int)
		self.__byte_address_type=int if i_currver==2 else int
		self.__range_iterator=xrange if i_currver==2 else range
		self.__first_pop_address=None
		self.__pop_byte_addresses={}
		self.__header_and_loci_byte_addresses={}
		self.__read_byte_addresses()
		return
	#end __init_object

	def __init_subsamples( self ):
		self.__indiv_subsamples={}
		self.__loci_subsamples={}
		self.__pop_subsamples={}
		return
	#end __delete_subsamples

	def __get_loci_count( self ):

		'''
		Get the count of loci as given by
		the header/loci info at top of genepop
		file.  Note that "Loci names can appear
		on separate lines, or on one line if separated by commas.
		'''

		i_loci_count=None

		'''
		Genepop file has one header line, then loci lines (up to first pop).
		'''
		i_loci_line_count=len( self.__header_and_loci_byte_addresses ) - 1

		'''
		If the file has only one loci line, then
		then all loci will be on this one line,
		comma-separated.
		'''

		if i_loci_line_count==1:
			IDX_LOCI_LINE=1
			o_orig_file=open( self.__filename, "rb" )
			l_byte_address=\
					self.__header_and_loci_byte_addresses[ \
												IDX_LOCI_LINE ]
			o_orig_file.seek( l_byte_address )
			v_loci_line=o_orig_file.readline()

			if type( v_loci_line )==bytes:
				v_loci_line=v_loci_line.decode( SYSENCODING )
			#end if type is bytes

			ls_loci=v_loci_line.split( "," )
			i_loci_count=len( ls_loci )
		else:
			i_loci_count=i_loci_line_count
		#end

		self.__loci_count=i_loci_count

		return
	#end __get_loci_count

	def __is_pop_line( self, b_line ):
		'''
		assumes b_line is a bytearray object
		representing a file line:
		'''
		b_lower=b_line.lower()

		#does the line begin with (case insensitive) "pop", followed
		#by an endline (unix/OSX (\n) or windows(\r\n) )?
		if b_lower.find( b'pop\n' ) == 0 or b_lower.find( b'pop\r\n' ) == 0:
			return True
		else:
			return False
		#end if pop line else not

		raise Exception( "in object instance of GenepopFileAddresses, def __is_pop_line, " \
				+ "should have returned in the if... else statement" )
	#end __is_pop_line

	def __is_indiv_line( self, b_line ):
		'''
		assumes b_line is a bytearray object
		assumes b_line comes from from a pop section
		(i.e. not from the header/loci-name
		part of the file)
		'''

		if b_line.find( b',' ) != -1:
			return True
		else:
			return False
		#end if line has a comma, else not
	#end __is_idiv_line

	def __get_byte_address_first_byte_in_line( self, o_file, b_line ):
		'''
		assumes b_line is a python bytearray object
		'''
		return self.__byte_address_type( o_file.tell() ) - self.__byte_address_type( len( b_line ) )
	#ene __get_byte_address_first_byte_in_line

	def __read_header_and_loci_entries( self ):
		'''
		finds byte addresses for each item preceeding
		the first "pop" line in the genepop file,
		inserts the byte addresses for each at the
		incremented int keys for member dict __header_and_loci_byte_addresses

		assumes one item (loci name, or loci list) to a line,
		so that each item will have one line only (see class description)
		'''
		o_file=open( self.__filename, 'rb' )
		i_item_count=0
		s_line=o_file.readline()

		while s_line:
			#for python3, need to convert the bytes object returnd by readline():
			b_line=bytearray( s_line )

			l_byte_address=self.__get_byte_address_first_byte_in_line( o_file, b_line )

			if self.__is_pop_line( b_line ):
				#no more loci lines, so record
				#the address to the firsrt "pop"
				#and return
				self.__first_pop_address=l_byte_address
				return
			else:
				self.__header_and_loci_byte_addresses[ i_item_count ] = l_byte_address
				i_item_count+=1
			#end if nonpop line . . . else
			s_line=o_file.readline()
		#end for line in file

		#with correct genepop line and code, should never reach this line:
		raise Exception( "In GenepopFileManager instance, def __read_header_and_loci, " \
				+ "found no \"pop\" line (case insensitive) in file, " \
				+ self.__filename + "." )

	#end __read_header_and_loci

	def __read_pops( self ):
		'''
		assumes object instance member __first_pop_address
		has a long value (int in python 3) that was set in
		def __read_header_and_loci that points to the first
		byte in the first line in the file that reads
		(case insensitive) as "pop"
		'''
		i_pop_count=0
		i_total_indiv_this_pop=0
		i_total_lines_this_indiv=0

		o_gpfile=open ( self.__filename, 'rb' )
		o_gpfile.seek( self.__first_pop_address )
		s_line=o_gpfile.readline()
		while  s_line:

			b_line=bytearray( s_line )

			l_byte_address=self.__get_byte_address_first_byte_in_line( o_gpfile, b_line )

			if self.__is_pop_line( b_line ):
				i_pop_count+=1
				#we record the "pop" line as the zeroth item in this pop
				#and we also assume it is a single-lined item:
				i_total_indiv_this_pop=0
				self.__pop_byte_addresses[ i_pop_count]={ i_total_indiv_this_pop: { 1 : l_byte_address } }
				i_total_lines_this_indiv=0
			elif self.__is_indiv_line:
				#is the first (if not only) line giving loci info for a single individual
				i_total_indiv_this_pop+=1
				i_total_lines_this_indiv=1
				self.__pop_byte_addresses[ i_pop_count ][ i_total_indiv_this_pop ]={ 1 : l_byte_address }
			else:
				i_total_lines_this_indiv +=1
				self.__pop_byte_addresses[ i_pop_count ][ i_total_indiv_this_pop ][ i_total_lines_this_indiv ] = l_byte_address
			#end if line starts with pop esle has comma (so is first line of indiv entry), else neither, so must be next line in loci
			#for individual
			s_line=o_gpfile.readline()
		#end for each line
	#end __read_pops

	def __read_byte_addresses( self ):
		self.__read_header_and_loci_entries()
		self.__read_pops()
		return
	#end __read_byte_addresses

	def __remove_individual_subsample( self, s_subsample_tag ):
		try:
			self.__indiv_subsamples.pop( s_subsample_tag )
		except Exception as oex:
			s_msg="In GenepopFileManager instance, def __remove_individual_subsample, " \
					+ "exception when trying to remove subsample, " \
					+ s_subsample_tag \
					+ ".  Exception thrown: " + str( oex ) + "."
			raise Exception( s_msg )
		#end try, except
		return
	#end __remove_individual_subsample

	def __combine_individual_subsamples_for_one_pop( self, ls_individual_subsample_tags, i_pop_number=1 ):
		set_indivs=set()
		for s_subsample_tag in ls_individual_subsample_tags:
			li_indiv_list=self.__get_list_indiv_numbers( i_pop_number=i_pop_number,
												s_indiv_subsample_tag=s_subsample_tag )
			set_indivs=set_indivs.union( li_indiv_list )
		#end for each subsample, add new individuals
		li_combined=list(set_indivs )
		li_combined.sort()
		return li_combined
	#end __combine_individual_subsamples_for_one_pop

	def __sample_individuals_randomly_from_one_pop( self, i_pop_number,
															i_sample_size,
																s_subsample_tag=None ):
			'''
			If s_subsample_tag arg is not None, instead of the whole indiv list
			for the given pop number, the subsample of indivs assoc with the tag
			is sampled.
			'''

			i_pop_size=None
			li_indiv_list=None

			if s_subsample_tag is None:
				i_pop_size=self.__get_count_indiv( self.__pop_byte_addresses[ i_pop_number ] )
				li_indiv_list=self.__get_list_indiv_numbers( i_pop_number=i_pop_number )
			else:

				i_pop_size=self.__get_count_indiv( \
						self.__indiv_subsamples[ s_subsample_tag ][ i_pop_number ] )
				li_indiv_list=self.__get_list_indiv_numbers( i_pop_number=i_pop_number,
														s_indiv_subsample_tag=s_subsample_tag )
			#end if no subsample tag else we want to sample a subsample

			i_sample_size=min( i_sample_size, i_pop_size )

			#we don't include zero as sample-able (zeo is the "pop" entry),
			#and 2nd arg range def is series max + 1:
			li_subsample=random.sample( li_indiv_list, i_sample_size )

			#to preserve the indiv list order in the orig file:
			li_subsample.sort()

			return li_subsample

	#end __sample_individuals_randomly_from_one_pop

	def __remove_n_individuals_randomly_from_one_pop( self, i_pop_number, i_n_to_remove ):
		li_indiv_list=self.__get_list_indiv_numbers( i_pop_number=i_pop_number )
		random.shuffle( li_indiv_list )
		li_shuffled_indiv_list_with_n_removed=li_indiv_list[ i_n_to_remove : ]
		li_shuffled_indiv_list_with_n_removed.sort()
		return li_shuffled_indiv_list_with_n_removed
	#end __remove_n_individuals_randomly_from_one_pop

	def __get_count_populations( self ):
		#recall the pops are numbered 1,2,3...N for N populations,
		#and that these numbers are the keys in the dict of byte addresses
		#giving the first line (the "pop" line ) of each pop
		return len( list(self.__pop_byte_addresses.keys()) )
	#end __get_count_populations

	def __get_pop_list( self, s_pop_subsample_tag = None ):

		li_pop_numbers=None

		if s_pop_subsample_tag is None:
			li_pop_numbers=list( self.__pop_byte_addresses.keys() )
		else:
			li_pop_numbers=self.__pop_subsamples[ s_pop_subsample_tag ]
		#end if no pop subsample tag else use subsample

		'''
		Not knowledgable enough to convince myself
		that keys will always be in order, though
		individual lists are sorted after random sampling.
		Also, I am not sure that sort will never matter, so:
		'''
		li_pop_numbers.sort()

		return li_pop_numbers
	#end __get_pop_list

	def __write_genepop_file_to_file_object( self, o_file_object,
			s_pop_subsample_tag=None,
			s_indiv_subsample_tag=None,
			s_loci_subsample_tag=None,
			i_min_pop_size=0 ):

		'''
		We convert all endlines to unix endlines. Found
		that in testing some dos-endline files, that
		simply writing whole lines from the file, then
		mixing with my subsampled loci lines, that i had
		genepop files with mixed endlines.  Note that testing
		these iwth NeEstimator shows they werre properly
		processed, but rather than test for which endline in
		orig, I'll simply use strip() and replace all whole
		line read/writes form the original with the unix endline.
		'''

		UNIX_ENDLINE="\n"

		#here we open the file without the 'b' flag, so we
		#will read string in both python 2 and 3
		'''
		2017_05_01.  Restoring the 'rb' flag, and adjusting code below to
		handle bytes objects, if the interpretor is python3.
		'''
		o_origfile=open( self.__filename, 'rb' )
		o_newfile=o_file_object

		li_header_and_loci_lines=None
		li_pop_numbers=None

		if s_pop_subsample_tag is None:
			li_pop_numbers=list(self.__pop_byte_addresses.keys())
		else:
			if s_pop_subsample_tag not in self.__pop_subsamples:
				s_msg="In GenepopFileManager instance, " \
								+ "def __write_genepop_file_to_file_object, " \
								+ "no population subsample with tag: " \
								+ s_pop_subsample_tag + "."
				raise Exception( s_msg )
			#end if no such population subsample tag
			li_pop_numbers=self.__pop_subsamples[ s_pop_subsample_tag ]
		#end if all pops to be written, else only those subsampled

		ls_subsampled_single_line_loci=None

		if s_loci_subsample_tag is None:
			li_header_and_loci_lines=list(self.__header_and_loci_byte_addresses.keys())
		elif len( self.__header_and_loci_byte_addresses ) == 2:
			'''
			This elsif conditional added 2017_12_05, to handle subsampled loci
			when the input file has all of its loci entered on line 2,
			with comma separators. Note that in this case the header and loci
			byte address dict will have only 2 keys, indicating two lines preceed
			the first "pop" entry.
			'''
			li_header_and_loci_lines=[ self.__header_and_loci_byte_addresses[ 0 ] ]
			o_origfile.seek( self.__header_and_loci_byte_addresses[ 1 ] )
			v_line_stripped=( o_origfile.readline() ).strip()
			if type( v_line_stripped ) == bytes:
				v_line_stripped=v_line_stripped.decode( SYSENCODING )
			#end if bytes type, decode
			ls_all_loci_on_one_line=v_line_stripped.split( "," )
			#loci subsampling is 1-based, but we have zero-based indexes
			#(e.g. loci 1 is the zeroth item in our list):
			ls_subsampled_single_line_loci=[ ls_all_loci_on_one_line[idx-1] \
					for idx in self.__loci_subsamples[ s_loci_subsample_tag ] ]
		else:
			l_header_address=self.__header_and_loci_byte_addresses[ 0 ]
			li_header_and_loci_lines=[ l_header_address ] \
					+ [ idx for idx in  self.__loci_subsamples[ s_loci_subsample_tag ] ]
		#end if we include all loci, else subsample

		#write header and loci list:
		for i_line_number in li_header_and_loci_lines:

			o_origfile.seek( self.__header_and_loci_byte_addresses[ i_line_number ] )

			'''
			In python 3, readline() will deliver a bytes object, which
			has the strip() method, just like a string in python 2.
			'''
			v_line_stripped=( o_origfile.readline() ).strip()

			if type( v_line_stripped ) == bytes:
				v_line_stripped=v_line_stripped.decode( SYSENCODING )
			#end if bytes type, decode

			o_newfile.write( v_line_stripped + UNIX_ENDLINE )

			o_newfile.flush()

		#end for each line in the header and loci section

		'''
		Added 2017_12_05, with new conditional above,
		when our original file uses single line loci entry.
		In this case only the header was written in the
		for loop above, and we now write the loci line
		that was computed in the conditional.
		'''
		if ls_subsampled_single_line_loci is not None:
			o_newfile.write( \
					",".join( ls_subsampled_single_line_loci ) \
					+ UNIX_ENDLINE )
		#end if we have subsampled, single-line loci entries.

		#write pops:
		for i_pop_number in li_pop_numbers:

			li_indiv_list=None
			i_tot_indiv=None

			if s_indiv_subsample_tag is None:
				li_indiv_list=list(self.__pop_byte_addresses[ i_pop_number ].keys())
				i_tot_indiv=self.__get_count_indiv( li_indiv_list )
			else:
				ddli_subsamples=self.__indiv_subsamples
				if s_indiv_subsample_tag not in ddli_subsamples:
					s_msg="In GenepopFileManager instance, def __write_genepop_file_to_file_object, " \
							"no indiv subsample with tag: " + s_indiv_subsample_tag
					raise Exception( s_msg )
				#end if no such tag

				if i_pop_number not in ddli_subsamples[ s_indiv_subsample_tag ]:
					s_msg="In GenepopFileManager instance, def __write_genepop_file_to_file_object, " \
							+ "subsample with tag " + s_indiv_subsample_tag \
							+ " has no subsample for population number " + str( i_pop_number ) + "."

					raise Exception( s_msg )
				#end if no such pop for this subsample
				li_indiv_list=ddli_subsamples[ s_indiv_subsample_tag ][ i_pop_number ]
				i_tot_indiv=self.__get_count_indiv( \
						ddli_subsamples[ s_indiv_subsample_tag ] [ i_pop_number ] )
			#end if no subsample, else subsample

			if i_tot_indiv >= i_min_pop_size:

				for i_indiv_number in li_indiv_list:

					#if this is the "pop" entry we can simply write it to file,
					#or, if we have no loci subsample, we can also write the indiv
					#entry as given in our original file:

					if i_indiv_number==GenepopFileManager.KEY_POP_ENTRY or s_loci_subsample_tag is None:
						for i_line_number in self.__pop_byte_addresses[ i_pop_number ][ i_indiv_number ]:
							o_origfile.seek( self.__pop_byte_addresses[ i_pop_number ]\
															[ i_indiv_number ][ i_line_number ] )
							v_line_stripped=( o_origfile.readline() ).strip()

							if type( v_line_stripped ) == bytes:
								v_line_stripped=v_line_stripped.decode( SYSENCODING )
							#end if bytes type, decode
							o_newfile.write( v_line_stripped + UNIX_ENDLINE )
						#end for each line number
					#otherwise we need to get loci via subsample:
					else:

						#Get ID separately:
						#We assume the complete individual ID is in line 1 of
						#the individual entry:
						s_id=self.__get_individual_id( o_origfile, self.__pop_byte_addresses[ i_pop_number ][ i_indiv_number ][ 1 ] )
						#Loci subsampled:
						s_loci=self.__get_loci_for_indiv( o_orig_file=o_origfile,
														i_pop_number=i_pop_number,
														i_indiv_number=i_indiv_number,
														s_loci_subsample_tag=s_loci_subsample_tag )

						if COMMA_DELIMITED_LOCI_LIST_HAS_LEADING_SPACE:
							s_line_to_write = s_id + ", " + s_loci
						else:
							s_line_to_write = s_id + "," + s_loci
						#end if we should insert a leading space in the loci list

						o_newfile.write( s_line_to_write + UNIX_ENDLINE )
					#end if no loci subsample, simply print orig entry, else get loci subsample
				#end for each individual number
			#end if num individuals in this pop at or over min
		#end for each pop number

		o_origfile.close()
		return
	#end __write_genepop_file_to_file_object

	def __get_count_indiv( self, iter_indiv_list ):
		'''
		To centralize the computation of counting
		num indiv -- in case the scheme of subracting
		one to account for the "pop" line changes.

		Passed iterable may be either a list of ints
		that give an indiv list, or the dictionary
		given by self.__byte_addresses[ pop_number],
		in which case we get the indiv numbers via
		the keys (we also sort to make sure we get
		the individual numbers in sorted order).
		'''

		if type( iter_indiv_list ) == dict:
			li_indiv_list=list( iter_indiv_list.keys() )
			li_indiv_list.sort()
		elif type( iter_indiv_list ) == list:
			li_indiv_list=iter_indiv_list
		else:
			s_msg="In GenepopFileManager instance, " \
					+ "def __get_count_indiv, "\
					+ "invalid type passed. " \
					+ "Expecting dictionary or list, " \
					+ "but received type: " \
					+ str( type( iter_indiv_list ) ) \
					+ "."
			raise Exception( s_msg )
		#end if arg is dict, else list, else error

		if li_indiv_list[ 0 ] != 0:
			s_msg="In GenepopFileManager instance, " \
						+ "def __get_count_indiv, " \
						+ "expected first item in list " \
						+ "to be zero, to indicate the " \
						+ "\"pop\" line.  List passed to def: " \
						+ str( li_indiv_list ) + "."
			raise Exception( s_msg )
		i_result=len( li_indiv_list )  - 1
		return i_result
	#end __get_count_indiv

	def __get_count_indiv_per_pop( self, s_indiv_subsample_tag=None, s_pop_subsample_tag=None ):

		li_counts=[]

		iter_pops_with_indiv_lists=None

		li_pop_numbers=self.__get_pop_list( s_pop_subsample_tag )

		if s_indiv_subsample_tag is None:
			iter_pops_with_indiv_lists=self.__pop_byte_addresses
		else:
			iter_pops_with_indiv_lists=self.__indiv_subsamples[ s_indiv_subsample_tag ]
		#end if we're counting all or a subsample

		for i_pop_number in li_pop_numbers:
			i_tot_this_pop=self.__get_count_indiv( iter_pops_with_indiv_lists[ i_pop_number ] )
			li_counts.append( i_tot_this_pop )
		#end for each pop number

		return li_counts
	#end get_count_indiv_per_pop

	def  writeGenePopFile( self, s_newfilename,
			s_pop_subsample_tag=None,
			s_indiv_subsample_tag=None,
			s_loci_subsample_tag=None,
			i_min_pop_size=0 ):

		if os.path.exists( s_newfilename ):
			s_msg="In GenepopFileManager object instance, " \
					+ "def writeGenePopFile(), can't write file, " \
					+ s_newfilename + ".  File already exists."
			raise Exception( s_msg )
		#end if file exists

		o_newfile=open( s_newfilename, 'w' )

		self.__write_genepop_file_to_file_object( o_newfile,
				s_pop_subsample_tag,
				s_indiv_subsample_tag,
				s_loci_subsample_tag,
				i_min_pop_size )

		o_newfile.close()

		return
	#end writeGenePopFile

	def printGenePopFile( self,
			s_pop_subsample_tag=None,
			s_indiv_subsample_tag=None,
			s_loci_subsample_tag=None,
			i_min_pop_size=0,
			o_outstream=sys.stdout ):

		self.__write_genepop_file_to_file_object( o_outstream,
													s_pop_subsample_tag,
													s_indiv_subsample_tag,
													s_loci_subsample_tag,
													i_min_pop_size )
		return
	#end printGenePopFile

	def getListIndividualNumbers( self, i_pop_number=1, s_indiv_subsample_tag=None):
		li_indiv=self.__get_list_indiv_numbers( i_pop_number, s_indiv_subsample_tag )
		return [ i_indiv for i_indiv in li_indiv ]
	#end def getListIndividualNumbers

	def __get_list_indiv_numbers( self, i_pop_number, s_indiv_subsample_tag = None ):
		li_indiv_numbers=[]

		if i_pop_number not in self.__pop_byte_addresses:
			s_msg="In GenepopFileManager object instance,  getListIndiv(), " \
					+ "no pop number, " + str( i_pop_number )
			raise Exception( s_msg )
		#end if no pop with number

		if s_indiv_subsample_tag is None:
			#if python 3 we can't sort the "dict_keys" object that py3 returns
			li_indiv_numbers=list( self.__pop_byte_addresses[ i_pop_number ].keys() )
			#These numbers should always be contiguous ints 0,1,2,3...totalIndivCount
			#We'll sort the keys (though tests with dicts with contiguous, increasing ints
			#show that keys are sorted already -- may not be guaranteed in all settings)
			li_indiv_numbers.sort()
		else:
			li_indiv_numbers=self.__indiv_subsamples[ s_indiv_subsample_tag ][ i_pop_number ]
		#end if no subsample else subsample
		#recall that all indiv lists include indiv "0", that is, the "pop" entry,
		#so we return only the numbers for tthe indivduals proper.
		return li_indiv_numbers [ 1 : ]
	#end __get_list_indiv_numbers

	def __get_list_loci_numbers( self, s_loci_subsample_tag=None ):
		li_loci_numbers=None
		if s_loci_subsample_tag is None:
			li_loci_numbers=list(range( 1, self.__loci_count + 1))
		else:
			li_loci_numbers=self.__loci_subsamples[ s_loci_subsample_tag ]
		#end if we have no loci subsample tag, else we have one

		return li_loci_numbers
	#end __get_list_loci_numbers

	def __get_individual_id( self, o_orig_file, l_address ):
		'''
		param o_filehandle is a file object open with 'rb' for reading
			the genepop file.  We ask for an open handle in order
			to avoid many open() commands if this def is called
			in a long loop

		'''

		s_indiv=None

		o_orig_file.seek( l_address )

		v_this_indiv_line=o_orig_file.readline()
		'''
		We assume the file was opened 'rb'.  If the interpretor
		is python3, then we need to convert the bytes object.
		'''
		if type( v_this_indiv_line ) == bytes:
			v_this_indiv_line=v_this_indiv_line.decode( SYSENCODING )
		#end if readline returned bytes, decode

		if "," not in v_this_indiv_line:
			s_msg="In GenepopFileManager instance, " \
						+ "def, __get_individual_id, " \
						+ "no comma found in entry line: " \
						+ v_this_indiv_line + "."
			raise Exception( s_msg )
		#end if no comma in line

		ls_this_indiv=v_this_indiv_line.split( "," )

		#possible to have a blank field in genepop
		#individual entry (i.e. line starts with comma),
		#in which case we return None
		if len( ls_this_indiv ) == 1 :
			s_indiv=None
		elif len( ls_this_indiv) == 2 :
			s_indiv= ls_this_indiv [ 0 ]
		else:
			s_msg = "In GenepopFileManager object instance,  getListIndiv(), " \
					+ "more than one comma found in individual listing.  Line reads: " \
					+ v_this_indiv_line.strip() + "\n"
			raise Exception( s_msg )
		#end if no id, else id, else parsing error

		return s_indiv

	#end __get_individual

	def __get_indiv_entry( self,
							o_orig_file,
							i_pop_number,
							i_indiv_number ):

		'''
		In the open file object given by o_orig_file,
		for each line in the individual given by i_indiv_number,
		and for the pop number given by i_pop_number,
		append to string var and return the string of
		concatenated entry lines.  Note that most often
		there will be only one entry line (id,loci-list).
		'''
		s_entry=""

		i_num_lines_this_indiv_entry=len( self.__pop_byte_addresses[ i_pop_number ][ i_indiv_number ] )

		if i_num_lines_this_indiv_entry==0:
			s_msg="In GenepopFileManager instance, " \
						+ "def __get_indiv_entry" \
						+ "found no entry for pop " \
						+ str( i_pop_number ) \
						+ " and individual number " \
						+ str( i_indiv_number ) + "."
			raise Exception( s_msg )
		#ene if no lines this pop/indiv

		li_entry_line_numbers_sorted=list( self.__pop_byte_addresses[ \
								i_pop_number ][ i_indiv_number ].keys() )

		li_entry_line_numbers_sorted.sort()

		for i_line_number in li_entry_line_numbers_sorted:
			l_address=self.__pop_byte_addresses[ i_pop_number ][ i_indiv_number ][ i_line_number ]
			o_orig_file.seek( l_address )

			'''
			For python 3, when reading in bytes from
			the 'rb' opened file:
			'''
			v_line=o_orig_file.readline()

			if type( v_line ) == bytes:
				v_line=v_line.decode( SYSENCODING )
			#end if we read in bytes

			s_entry+=v_line

		#end for each line in entry

		return s_entry
	#end __get_indiv_entry

	def __get_loci_for_indiv( self,
				o_orig_file,
				i_pop_number,
				i_indiv_number,
				s_loci_subsample_tag=None ):

		'''
		In the open file given by o_orig_file,
		for the population given by i_pop_number,
		and the individual given by i_indiv_number,
		return the loci, either the original set,
		or, if not None, the subsampled loci given
		by s_loci_subsample_tag.
		'''
		IDX_LOCI=1

		s_loci=""

		s_entry=self.__get_indiv_entry( o_orig_file,
										i_pop_number,
										i_indiv_number )

		ls_id_and_loci=s_entry.split( "," )

		if len( ls_id_and_loci ) != 2:
			s_msg="In GenepopFileManager instance, " \
						+ "def __get_loci_for_indiv, " \
						+ "No single-comma delimiter found " \
						+ "separating id and loci in entry " \
						+ "line or lines: " + s_entry + "."
			raise Exception( s_msg )
		#end if not a 2-item list, then can't tell id from loci

		s_loci_line_or_lines=ls_id_and_loci[ IDX_LOCI ]

		if s_loci_subsample_tag is None:
			s_loci=s_loci_line_or_lines
		else:
			'''
			We must split the loci to get
			the subsamples.  See format
			spec at web address listed in
			comments head of this file.
			'''

			#To combine multiple loci lines,
			#replace newlines with spaces:
			s_loci_line=s_loci_line_or_lines.replace( "\n", " " )

			'''
			Seperator of loci can be either space
			or tab.  Python split() with no arg
			splits on either and deletes empty
			strings (i.e. 2 ormore "blank" delimiters
			in a row )
			'''

			ls_loci=s_loci_line.split()

			li_loci_numbers=\
					self.__loci_subsamples[ \
								s_loci_subsample_tag ]

			#Note that loci numbers are 1-based, so we subtract
			#one to find them in our loci list:

			s_loci=" ".join( [ ls_loci[ i_num-1 ] for i_num in li_loci_numbers ] )

		#end if loci not subsampled else subsampled

		return s_loci
	#end getLociForIndiv

	def __get_range_loci_nums( self,
									i_min_loci_position,
									i_max_loci_position,
									b_truncate_max_to_total ):
		'''
		Returns a range object (in python3, not synonymous
		with a list type).  Checks for valid range.  if the
		b_truncate_max_to_total is True, then it truncates
		to the max loci number N (1,2,3...N for N total loci).
		'''

		li_range_loci_numbers=None

		'''
		Allows use of any value for max
		without knowing how many loci are
		available.  We still sample from
		correct range:
		'''
		if b_truncate_max_to_total:
			i_max_loci_position=min( self.__loci_count, i_max_loci_position )
		#end if truncate

		b_min_out_of_range=i_min_loci_position < 1 or \
				i_min_loci_position >= i_max_loci_position \
				or  i_min_loci_position >= self.__loci_count

		b_max_out_of_range=i_max_loci_position > self.__loci_count

		if b_min_out_of_range or b_max_out_of_range:
			s_msg="In GenepopFileManager instance, " \
						+ "def __get_range_loci_nums, " \
						+ "start-end loci range requested, " \
						+ str( i_min_loci_position ) \
						+ "-" + str( i_max_loci_position ) \
						+ ", out of range for file, " \
						+ self.__filename \
						+ ", with loci total, " \
						+ str( self.__loci_count ) + "."
			raise Exception( s_msg )
		#end if requested range invalid

		li_range_loci_numbers=list(range( i_min_loci_position, ( i_max_loci_position + 1 )))

		return li_range_loci_numbers
	#end def __get_range_loci_nums

	def subsampleLociByRangeAndMax( self, i_min_loci_position,
			i_max_loci_position,
			s_loci_subsample_tag,
			i_min_total_loci=None,
			i_max_total_loci=None,
			b_truncate_max_to_total=True ):

		'''
		Generate list of integers that stand for loci positions,
		and as such allow for printing all or a subset of the loci
		for a given pop/indiv.  Given we have a set of N loci l_x in a
		genepop file listed in order l_1, l_2, l_3...l_N.

		param i_min_loci_position : integer giving lower index for range of loci numbers
		param i_max_loci_position : integer giving upper index for range of loci numbers
		param i_min_total: sample at least this many loci, or throw error
		param i_max_total: sample at most this many loci, randomly select if range exceeds
		param b_truncate_max_to_total : boolean, if True, then, when the param i_max_loci_position
			                            exceeds the total available loci, N, we set it to N.
										If false then we keep the max position, in order to
										throw an error if it exceeds N (see def __get_range_loci_nums),
										default is True.

		'''

		self.__loci_subsamples[ s_loci_subsample_tag ]=[]

		o_range_to_sample=self.__get_range_loci_nums( i_min_loci_position,
															i_max_loci_position,
															b_truncate_max_to_total )

		i_loci_range_interval_size=len( o_range_to_sample )

		i_sample_size=i_loci_range_interval_size

		if i_min_total_loci is not None:
			if i_sample_size < i_min_total_loci:
				s_msg="In GenepopFileManager instance, def " \
							+ "subsampleLociByRangeAndMax " \
							+ "loci sample size, " \
							+ str( i_sample_size )  \
							+ ", is less than the minimum " \
							+ "given by the passed argument: "  \
							+ str( i_min_total_loci ) + "."
				raise Exception( s_msg )
			#end if sample size under min
		#end if min total loci is not None

		if i_max_total_loci is not None:
			i_sample_size=\
					min( i_loci_range_interval_size, i_max_total_loci )
		#end if we have a value for max total loci

		li_subsample=random.sample( o_range_to_sample, i_sample_size )

		li_subsample.sort()

		self.__loci_subsamples[ s_loci_subsample_tag ]=li_subsample

		return
	#end def subsampleLociByRangeAndMax

	def subsampleLociByRangeAndProportion( self,
			i_min_loci_position,
			i_max_loci_position,
			s_loci_subsample_tag,
			f_proportion_of_total_loci,
			i_min_total_loci=None,
			b_truncate_max_to_total=True ):

		'''
		Generate list of integers that stand for loci positions,
		and as such allow for printing the given proportion of loci
		with the indices range given.  Flag b_truncate_max_to_total
		when True will truncate the max-loci-position param to the
		max loci number, when the given value exceeds.  Otherwise,
		if max is out of range an error will be thrown.

		'''

		self.__loci_subsamples[ s_loci_subsample_tag ]=[]

		o_range_to_sample=self.__get_range_loci_nums( i_min_loci_position,
															i_max_loci_position,
															b_truncate_max_to_total )

		i_loci_range_interval_size=len( o_range_to_sample )

		f_proportion_of_sample=float (i_loci_range_interval_size  ) * f_proportion_of_total_loci

		i_sample_size=int( round(  f_proportion_of_sample ) )

		if i_min_total_loci is not None:
			if i_sample_size < i_min_total_loci:
				s_msg="In GenepopFileManager instance, def " \
							+ "subsampleLociByRangeAndProportion, " \
							+ "loci sample size, " \
							+ str( i_sample_size )  \
							+ " is less than the minimum " \
							+ "given by the passed argument: "  \
							+ str( i_min_total_loci ) + "."
				raise Exception( s_msg )
			#end if sample size under min
		#end if i_min_total_loci is not None

		li_subsample=random.sample( o_range_to_sample, i_sample_size )

		li_subsample.sort()

		self.__loci_subsamples[ s_loci_subsample_tag ]=li_subsample

		return

	#end subsampleLociByRangeAndProportion

	def getListIndividuals( self, i_pop_number=1, s_indiv_subsample_tag = None ):

		ls_individuals=[]

		li_indiv_numbers= self.__get_list_indiv_numbers( i_pop_number, s_indiv_subsample_tag )
		o_orig_file = open( self.__filename, 'rb' )

		for i_this_indiv in li_indiv_numbers:

			l_address=self.__pop_byte_addresses[ i_pop_number ][ i_this_indiv ][1]
			o_orig_file.seek( l_address )
			v_this_indiv_line=o_orig_file.readline()

			'''
			2017_05_01. For python3, we need to convert
			the bytes read-in to a string type:
			'''
			if type( v_this_indiv_line )==bytes:
				v_this_indiv_line=v_this_indiv_line.decode( SYSENCODING )
			#end if bytes, decode

			ls_this_indiv=v_this_indiv_line.split( "," )

			#possible to have a blank field in genepop
			#individual entry (i.e. line starts with comma),
			#in which case we'll simply return the number
			#indicating this is the nth individual in the list
			if len( ls_this_indiv ) == 1 :
				s_msg= "In GenepopFileManager instance, " \
						+ "def getListIndiv, " \
						+ "Expecting commat delimiter between " \
						+ "ID and Loci, but comma found on id line: " \
						+ s_this_indiv_line.strip() + "."
				raise Exception( s_msg )
			elif len( ls_this_indiv) == 2 :
				ls_individuals.append( ls_this_indiv [ 0 ] )
			else:
				s_msg = "In GenepopFileManager object instance,  getListIndiv(), " \
						+ "more than one comma found in individual listing.  Line reads: " \
						+ s_this_indiv_line.strip() + "\n"
				raise Exception( s_msg )
		#end for each index in list of individuals for pop

		o_orig_file.close()

		return ls_individuals
	#end getListIndividuals

	def getListPopulationNumbers( self, s_pop_subsample_tag=None ):

		li_pop_numbers=self.__get_pop_list( s_pop_subsample_tag )

		#to secure the list,
		#return a copy of
		return [ i_num for i_num in li_pop_numbers ]
	#end getListPopulationNumbers

	def getListEmptyPopulationNumbers( self,
			s_pop_subsample_tag=None,
			s_indiv_subsample_tag=None ):

		li_empty_pops=[]

		li_pop_numbers=self.__get_pop_list( s_pop_subsample_tag )

		for i_pop_number in li_pop_numbers:
			li_indiv_numbers=None

			if s_indiv_subsample_tag is None:
				li_indiv_numbers=list(self.__pop_byte_addresses[ i_pop_number ].keys())
			else:
				li_indiv_numbers=self.__indiv_subsamples[ s_indiv_subsample_tag ] [ i_pop_number ]
			#end if we son't have an indiv subsample tag else we do

			#subtract one from the indiv list
			#to account for the "pop" line:
			i_num_indivs=self.__get_count_indiv( li_indiv_numbers )
			if i_num_indivs == 0:
				li_empty_pops.append( i_pop_number )
			#end if no indiv in this pop
		#end for each pop number

		return li_empty_pops
	#end getListEmptyPops

	def subsampleIndividualsRandomlyByProportion( self, f_proportion_to_sample, s_subsample_tag ):
		'''
		for each "pop" in the original genepop file,
		randomly select round( total_indiv_in_this_pop * f_proportion_to_sample )

		if zero is the subsample size, then an empty list will be assoc with the pop number
		stores the subset list of individuals as ints, ordered as they are in the orig, as
		self.__indiv_subsamples[ s_subsample_tag ][ i_pop_number ]
		'''

		self.__indiv_subsamples[ s_subsample_tag ]={}

		for i_pop_number in self.__pop_byte_addresses:

			i_pop_size=self.__get_count_indiv( self.__pop_byte_addresses[ i_pop_number ] )

			i_sample_size=int( round( i_pop_size * f_proportion_to_sample ) )

			li_subsample=self.__sample_individuals_randomly_from_one_pop( i_pop_number,i_sample_size )

			#we always include the "pop" entry, and we sort the subsample
			self.__indiv_subsamples[ s_subsample_tag ][ i_pop_number]= \
					 [ 0 ] +  li_subsample
		#end for each pop number

		return
	#end subsampleIndividualsRandomlyByProportion

	def subsampleNIndividualsRandomlyFromEachPop( self, i_n, s_subsample_tag ):

		'''
		for each "pop" in the original genepop file,
		randomly select N individuals

		if N is larger than a given population, an error is thrown
		by random.sample.  The sampled pops are stored as
		the subset list ints, ordered as they are in the orig, as
		self.__indiv_subsamples[ s_subsample_tag ][ i_pop_number ]
		'''

		self.__indiv_subsamples[ s_subsample_tag ] = {}

		for i_pop_number in self.__pop_byte_addresses:

			li_subsample=self.__sample_individuals_randomly_from_one_pop( i_pop_number, i_n )

			#we always include the "pop" entry, and we sort the subsample
			self.__indiv_subsamples[ s_subsample_tag ][ i_pop_number]= \
					 [ 0 ] +  li_subsample
		#end for each pop number
	#end subsampleNIndividualsRandomlyFromEachPop

	def subsampleLociByName( self, ls_loci_names, s_subsample_tag ):
		raise Exception( "not implemented" )
		return
	#end subsampleLociByName

	def subsampleIndividualsMinusRandomNFromEachPop( self, i_n_remove, s_subsample_tag ):

		self.__indiv_subsamples[ s_subsample_tag ] = {}

		for i_pop_number in self.__pop_byte_addresses:
			li_sample=self.__remove_n_individuals_randomly_from_one_pop( i_pop_number, i_n_remove )
			self.__indiv_subsamples[ s_subsample_tag ][ i_pop_number ] = [0] + li_sample
		#end for each pop number

		return
	#end subsampleIndividualsMinusRandomNFromEachPop

	def subsampleIndividualsLeaveNthOutFromPop( self, i_n,
													s_indiv_subsample_tag ):
		'''
		Remove the nth of M individuals,
		where individuals are numbered 1,2,3...M

		Requires value for i_n, 0<i_n.  If i_n > pop size,
		we enter an empty list for the pop for ths given subsample tag.
		'''

		self.__indiv_subsamples[ s_indiv_subsample_tag ] = {}

		li_all_population_numbers=self.__get_pop_list()

		li_indiv_list_copy=None

		for i_pop_number in li_all_population_numbers:

			li_indiv_list=self.__get_list_indiv_numbers( i_pop_number, s_indiv_subsample_tag=None )

			i_pop_size=len( li_indiv_list )

			if  i_n < 1:
				s_msg="In GenepopFileManager instance, def " \
						+ "subsampleIndividualsLeaveNthOutFromPop, " \
						+ "invalid N.  With " + str( i_pop_size ) \
						+ " individuals, and N = " \
						+ str( i_n ) + "."

				raise Exception ( s_msg )
			elif i_n>i_pop_size:
				#Can't leave out nth indiv, since n too large,
				#so we assign an emtpy list for this pop:
				li_indiv_list_copy=[]

			else:
				#want a copy of the list:
				li_indiv_list_copy = [ i_indiv for i_indiv in li_indiv_list ]
				li_indiv_list_copy.remove( i_n )
			#end if n < 1, n>pop size, else in range

			#to any sample we always add the zeroth
			#indiviudal number, which is the "pop"
			#entry
			self.__indiv_subsamples[ s_indiv_subsample_tag ] [ i_pop_number ] = \
					[0] + li_indiv_list_copy
		#end for each pop number
		return
	#end subsampleIndividualsLeaveNthOutFromPop

	def subsampleIndividualsByIdCriteria( self,
											o_genepop_indiv_id_fields,
											o_genepop_indiv_criteria,
											s_subsample_tag,
											i_min_pop_sample_size=None,
											i_max_pop_sample_size=None ):
		'''
		param o_genepop_indiv_id_fields, an instance of GenepopIndivIdFields

		param o_genepop_indiv_criteria, a GenepopIndivCriteria instance,
			itself a list of GenepopIndivCriterion instances, with
			the ability to perform all criterion tests and return True or False

		Assumes that the list of individual (ids)
		from call to getListIndividuals gives ids
		in the same order as they occur in the
		original genepop file.
		'''
		self.__indiv_subsamples[ s_subsample_tag ]={}

		for i_pop_number in self.__pop_byte_addresses:

			ls_individuals=self.getListIndividuals( i_pop_number )

			li_indiv_numbers=[]

			for idx in range( len( ls_individuals ) ):
				s_this_id=ls_individuals[ idx ]
				o_genepop_id_vals=GenepopIndivIdVals( s_this_id,
											o_genepop_indiv_id_fields )

				o_genepop_individual_id=GenepopIndividualId( o_genepop_id_vals, o_genepop_indiv_criteria )

				if o_genepop_individual_id.allCriteriaAreTrue():

					#recall that the indices into individuals
					#start with "1", since the 0th individual
					#is the "pop" entry itself:
					li_indiv_numbers.append( idx + 1 )
				#end if criteria all True, add indiv to subsample
			#end for each individual

			#we always include the "pop" entry, and we sort the subsample
			self.__indiv_subsamples[ s_subsample_tag ][ i_pop_number]= \
												 [ 0 ] +  li_indiv_numbers

			if i_max_pop_sample_size is not None:

				i_indiv_count=self.__get_count_indiv( self.__indiv_subsamples[ s_subsample_tag ][ i_pop_number ] )

				if i_indiv_count> i_max_pop_sample_size:

					li_reduced_indiv_list=self.__sample_individuals_randomly_from_one_pop( \
															i_pop_number=i_pop_number,
															i_sample_size=i_max_pop_sample_size,
															s_subsample_tag=s_subsample_tag )

					self.__indiv_subsamples[ s_subsample_tag ][ i_pop_number ] = \
														[0] + li_reduced_indiv_list
				#end if we're over the max and need to randomly select
			#end if we have a max value for indiv count

			#Reduce to no individuals if we have a min pop size and our sampled pop
			#is under the minimum.
			if i_min_pop_sample_size is not None:
				i_indiv_count=self.__get_count_indiv( self.__indiv_subsamples[ s_subsample_tag ][ i_pop_number ] )
				if i_indiv_count < i_min_pop_sample_size:
					self.__indiv_subsamples[ s_subsample_tag ][ i_pop_number ] = [0]
				#end if under min count, reduce to no individuals
			#end if we have a minimum indiv count number
		#end for each pop number

		return
	#end subsampleIndividualsByIdCriteria

	def subsampleIndividualsByNumberList( self, dli_individual_numbers,
			s_subsample):
		'''
		param dli_individual_numbers, dictionary keyed to population
				numbers.  For each key the value is an ordered list
				of increasing ints i, 1<=i<=N, that make a set of individual
				numbers for the population in the kay.

		Note that this subsampling will put an empty list in populations whose
		numbers exist in the genepop file, but not among the dictionary keys.
		'''

		li_all_population_numbers=self.__get_pop_list()
		self.__indiv_subsamples[ s_subsample ] = {}
		for i_pop_num in li_all_population_numbers:
			'''
			If caller's dictionary keys lack this pop number
			the subsample is an empty list, except for the
			"pop" line (item 0 in any given indiv list):
			'''
			li_this_indiv_list=[0]
			if i_pop_num in dli_individual_numbers:

				#all indiv subsample lists include the zeroth item,
				#the "pop" entry
				li_this_indiv_list =  [ 0 ] +  dli_individual_numbers[ i_pop_num ]

				#we always store sorted indiv ordinals
				#in the subsample lists:
				li_this_indiv_list.sort()
			#end if we have a list of indiv for this pop
			self.__indiv_subsamples[ s_subsample ] [ i_pop_num ] = li_this_indiv_list
		#end for each pop number in file
		return
	#end subsampleIndividualsByNumberList

	def subsampleIndividualsByPopSize( self, i_min_pop_size, i_max_pop_size, s_subsample ):
		'''
		For each pop, sample is identical to original, unless
		its indiv count is under i_min_pop_size, in which case
		it is reduced to zero individuals, or its indiv count
		exceeds i_max_pop_size, in which case i_max_pop_size
		individuals aret randomly sampled .
		'''

		li_all_population_numbers=self.__get_pop_list()
		self.__indiv_subsamples[ s_subsample ] = {}

		for i_pop_number in li_all_population_numbers:

			li_sample_this_pop=None

			i_indiv_count=self.getIndividualCount( i_pop_number )

			if i_indiv_count < i_min_pop_size:
				li_sample_this_pop=[0]

			else:
				if i_indiv_count <= i_max_pop_size:
					li_sample_this_pop=[ 0 ] + self.__get_list_indiv_numbers( i_pop_number )
				else:
					li_sample_this_pop= [ 0 ] + \
							self.__sample_individuals_randomly_from_one_pop( \
																 i_pop_number=i_pop_number,
																 i_sample_size=i_max_pop_size )
				#end if indiv count under or equal to max, else over
			#end if under min, no indiv sampled, else sample

			self.__indiv_subsamples[ s_subsample ][ i_pop_number ]=li_sample_this_pop
		#end for each pop
		return
	#end subsampleIndividualsByPopSize

	def getListTuplesValuesIndivIdFields( self,
							o_genepop_indiv_id_fields,
							ls_field_names ):

		'''
		Need a method to get the set (i.e. unique tuple) of
		existing values for seom combo of ID fields
		(may be only one field).  This would
		be step one in a generalized implementation of Tiago's
		sampleIndivsRelated.py code. Once we have the set of tuples
		giving existing values, we can then use criteria matching
		to subsample pops by testing each indiv for (a) matching
		field value(s), using def subsampleIndividualsByIdCriteria.
		Note that we can add criteria to the tests, for example,
		to add age==1 to a criteria that also requires a father==i
		and mother==j match, each i and j supplied by this def.

		arg o_genepop_indiv_id_fields, an instance of GenepopIndivIdFields
		arg ls_field_names, a list of strings that ell which of the field names
				in o_genepop_indiv_id_fields gives the fields to group to find
				all uniq value combos.

		As of 2016_09_08, this not yet implemented
		'''

		li_pop_numbers=self.__get_pop_list()
		set_tuples_values=set()

		for i_pop_number in li_pop_numbers:
			ls_indiv_ids_this_pop=self.getListIndividuals( i_pop_number=i_pop_number )

			for s_id in ls_indiv_ids_this_pop:

				o_genepop_id_vals=GenepopIndivIdVals( s_id, o_genepop_indiv_id_fields )

				tv_vals_this_id=tuple()

				for s_field_name in ls_field_names:
					v_val_this_field=o_genepop_id_vals.getVal( s_field_name )
					tv_vals_this_id += ( v_val_this_field, )
				#end for each field, add to tuple

				set_tuples_values.union( { tv_vals_this_id } )

			#end for indiv id in this pop
		#end for each pop in file

		return list( set_tuples_values )

	#end getListTuplesValuesIndivIdFields

	def subsamplePopulationsByList( self, li_pop_numbers, s_subsample_tag ):

		i_total_pops=self.__get_count_populations()

		li_pop_numbers.sort()

		i_min_sampnum=min( li_pop_numbers )
		i_max_sampnum=max( li_pop_numbers )

		if i_min_sampnum >= 1 and i_max_sampnum <= i_total_pops:
			#we want to copy the list -- not just have a reference
			self.__pop_subsamples[ s_subsample_tag ] = [ idx for idx in li_pop_numbers ]
		else:
			s_msg="In GenepopFileManager instance, def subsamplePopulationsByList, " \
					+ "genepop file has " + str( i_total_pops ) + " populations.  " \
					+ "sample list ranges out of bounds, with min: " + str( i_min_sampnum ) \
					+ ", and max: " + str( i_max_sampnum )

			raise Exception( s_msg )
		#end if all pop numbers in range, else error

		return
	#end subsamplePopulationsByList

	def getIndividualCounts( self, s_pop_subsample_tag=None, s_indiv_subsample_tag=None ):

		'''
		Returns a list of individual counts (total individuals) for each of N populations
		numbered 1,2,3,...N, representing the "pop" entries in order in the genepop file
		to which this GenepopFileManager instance corresponds.

		If s_pop_subsample_tag is not None, then the list will contain counts for each
		population in some subset of the original N, as listed by population nunber
		list associated with the population subsample tag.

		If s_indiv_subsample_tag is not None, then the counts will be for the individuals
		in each pop as subsampled.
		'''
		li_counts=None

		li_counts=self.__get_count_indiv_per_pop( s_indiv_subsample_tag, s_pop_subsample_tag )

		return li_counts

	#end getIndividualCounts

	def getIndividualCount( self, i_pop_number, s_indiv_subsample_tag=None ):
		'''
		Expects 1-based population number.
		Returns the individual count for the ith of N populations, i=1,2,3...N
		If s_indiv_subsample_tag is supplied, then the counts will
		reflect those of the individuals in the ith population after
		the sampling named by the tag.
		'''

		i_total_pops_in_file=self.__get_count_populations()

		if i_pop_number < 1 or i_pop_number > i_total_pops_in_file:

			s_msg="In GenepopFileManager instance, def getIndividualCount, " \
					+ "invalid population number for file with total populations, " \
					+ str( i_total_pops_in_file ) \
					+ ".  Pop number requested: " + str( i_pop_number ) + "."

			raise Exception( s_msg )

		#end if  invalid population number

		li_counts=None

		li_counts=self.__get_count_indiv_per_pop( s_indiv_subsample_tag )
		#end if we get counts from subsample

		return li_counts[ i_pop_number - 1 ]
	#end getIndividualCount

	def combineIndividualSubsamples( self, ls_subsample_tags,
							s_tag_for_combined_subsample ):

		li_population_numbers=self.__get_pop_list()
		self.__indiv_subsamples[ s_tag_for_combined_subsample ] = {}

		for i_pop in li_population_numbers:
			li_combined_indiv_list= \
						self.__combine_individual_subsamples_for_one_pop( \
															ls_subsample_tags,
															i_pop_number=i_pop )
			self.__indiv_subsamples[ s_tag_for_combined_subsample ][ i_pop ] = \
					[ 0 ] + li_combined_indiv_list
		#end for each pop, combine

		return
	#end combineIndividualSubsamples

	def removeIndividualSubsamples( self, ls_subsample_tags ):
		for s_subsample_tag in ls_subsample_tags:
			self.__remove_individual_subsample( s_subsample_tag )
		#end for each subsample tag, remove
		return
	#end removeIndividualSubsamples

	@property
	def pop_total( self ):
		return self.__get_count_populations()
	#end getNumberOfPops

	@property
	def loci_total( self ):
		return self.__loci_count
	#end loci_total

	@property
	def population_subsample_tags( self ):
		'''
		so user can iterate over populations
		as subsampled
		'''
		return [ s_tag for s_tag in self.__pop_subsamples ]
	#end population_subsample_tags

	@property
	def indiv_subsample_tags( self ):
		'''
		so user can iterate over subsamples,
		we return a list of their tags:
		'''
		return [ s_tag for s_tag in self.__indiv_subsamples ]
	# end def indiv_subsample_tags

	@property
	def loci_subsample_tags( self ):
		'''
		so user can iterate over subsamples,
		we return a list of their tags:
		'''
		return [ s_tag for s_tag in self.__loci_subsamples ]
	#end def loci_subsample_tags

	@property
	def original_file_name( self ):
		return self.__filename
	#end original_file_name

	@original_file_name.setter
	def original_file_name( self, s_filename ):
		'''
		this setter essentially creates a new object
		based on the genepop file with the filename
		given by param s_filename
		'''
		self.__filename=s_filename
		self.__setup_addresses( s_filename )
		self.__init_subsamples()
		return
	#end original_file_name (setter)

	def __get_allele_counts( self, s_pop_subsample_tag=None,
										s_indiv_subsample_tag=None,
										s_loci_subsample_tag=None,
										b_skip_loci_with_parial_data=True ):

		'''
		For each pop in the file (or as filtered by pop subsample tag), count the
		alleles.  When a loci has no data for an individual, counts are unchanged.
		When a loci has partial, the flag param is used do either leave unchanged
		or count the non-missing allele (see comment below).
		param tags give keys to subsamples of pops, individuals, or loci.
		param b_skip_loci_with_parial_data is meant to imitate the Genepop program,
			  which, when computing allele frequencies on diploid entries, apparently
			  skips non-zero alleles when the other is zero (missing).  Default
			  True will imitate, false will only skip alleles when both alleles are
			  zero (missing).
		'''

		MISSING_ALLELE=0

		#For assertion messages:
		s_location_msg="in GenepopFileManager instance " \
								+ "def __get_allele_counts, "
		li_pop_nums=None
		li_indiv_nums=None
		li_loci_nums=None

		dddi_allele_counts_by_pop_by_loci={}

		o_this_genepop_file=open( self.__filename, 'r' )

		li_pop_nums=self.__get_pop_list( s_pop_subsample_tag )
		li_loci_nums=self.__get_list_loci_numbers( s_loci_subsample_tag )
		i_total_loci=len( li_loci_nums )

		for i_pop_number in li_pop_nums:

			dddi_allele_counts_by_pop_by_loci[ i_pop_number ] = { i_loci_number : {}
																	for i_loci_number in li_loci_nums }

			li_indiv_nums=self.__get_list_indiv_numbers( i_pop_number, s_indiv_subsample_tag )

			for i_indiv_number in li_indiv_nums:

				s_loci_as_genepop_entry=self.__get_loci_for_indiv( o_this_genepop_file,
																	i_pop_number,
																		i_indiv_number,
																		s_loci_subsample_tag )

				ls_loci_list=s_loci_as_genepop_entry.split()
				assert i_total_loci == len( ls_loci_list ), s_location_msg \
											+ "individual loci list total, " \
											+ str( len( ls_loci_list ) ) \
											+ ", differs from expected total loci, " \
											+ str( i_total_loci ) \
											+ "using loci subsample with tag: " \
											+ str( s_loci_subsample_tag ) + "."

				for idx in range( i_total_loci ):

					s_this_loci=ls_loci_list[ idx ]
					i_this_loci_number=li_loci_nums[ idx ]

					#The sub-dictionary that holds running allele counts for this pop and loci:
					ddi_alleles_this_pop_and_loci= \
								dddi_allele_counts_by_pop_by_loci[i_pop_number ][ i_this_loci_number ]
					i_char_count=len( s_this_loci )

					#If loci entry diploid, then it will have at least 4 characters,
					#and no more than 6:
					assert i_char_count > 3 and i_char_count <= 6, s_location_msg \
										+ "loci entry, " + s_this_loci + ", " \
										+ "has character count: " \
										+  str( i_char_count ) + ".  " \
										+ "Current version requires diploid loci."

					#If the loci entry is diploid, then its length must be halvable:
					assert i_char_count % 2 == 0,  s_location_msg \
										+ "loci entry, " + s_this_loci \
										+ ", character count not divisible by two."

					i_chars_per_allele=old_div(i_char_count,2)

					try:
						i_allele_1=int( s_this_loci[ 0:i_chars_per_allele ] )
						i_allele_2=int( s_this_loci[ i_chars_per_allele:i_char_count ] )
					except ValueError as ove:
						raise Exception( s_location_msg  \
										+ "loci entry, " + s_this_loci \
										+ ", can't be converted " \
										+ "into 2 integers. Error: " + str( ove ) )
					#end try...except

					li_alleles=[ i_allele_1, i_allele_2 ]

					if sum( li_alleles ) == MISSING_ALLELE \
							or ( ( MISSING_ALLELE in li_alleles ) \
									and b_skip_loci_with_parial_data ):
						continue
					#end if missing all data, or missing at least one and flag says skip.

					for i_this_allele in li_alleles:
						if i_this_allele != MISSING_ALLELE:
							if i_this_allele not in ddi_alleles_this_pop_and_loci:
								ddi_alleles_this_pop_and_loci[ i_this_allele ] = 1

							else:
								ddi_alleles_this_pop_and_loci[ i_this_allele ] += 1
							#end if allele already recorded, else not
						#end if not a missing allele --(zero allele (missing data) is
						#possible when the b_skip_loci_with_parial_data flag is False.
					#end for each allele
				#end for each loci
			#end for each individual
		#end for each pop

		return dddi_allele_counts_by_pop_by_loci

	#end __get_allele_counts

	def getAlleleCounts( self, s_pop_subsample_tag = None,
									s_indiv_subsample_tag = None,
									s_loci_subsample_tag = None,
									b_skip_loci_with_parial_data=True ):

		dddi_allele_counts_by_pop_by_loci=self.__get_allele_counts( s_pop_subsample_tag,
																		s_indiv_subsample_tag,
																		s_loci_subsample_tag,
																		b_skip_loci_with_parial_data )

		return dddi_allele_counts_by_pop_by_loci
	#end getAlleleCounts

	@property
	def header( self ):
		'''
		This getter was added 2017_02_12 to allow clients to get
		the header string, and so look for an Nb/Ne value in the
		header text.  The current implementation stores this
		value for the Nb estimation in the header line
		as the final text, reading "nbne=<value>".  See
		pgdriveneestimator.py, def
		get_nbne_ratio_from_genepop_file_header.
		'''
		o_origfile=open( self.__filename, 'r' )
		o_origfile.seek( self.__header_and_loci_byte_addresses[0] )
		s_line_stripped=( o_origfile.readline() ).strip()
		return s_line_stripped
#end class GenepopFileManager

if __name__ == "__main__":
#	test the code
#	mydir=os.path.sep + os.path.join( "home","ted","documents","source_code","python","negui","temp_data","genepop_from_brian_20160506" )
#	restag="py" + str( sys.version_info.major ) + "test"
#	filecount=0
#	for myfile in [ "AlpowaCreekBY2006_14.txt", "AsotinCreekBY2006_104.txt", "BigCreekBY2006_29.txt" ]:
#		filecount+=1
#		mygpfile=mydir + os.path.sep + myfile
#
#		o_gp=GenepopFileManager( mygpfile )
#		lf_props=[ 0.10, 0.20, 0.30 ]
#		for f_prop in lf_props:
#			o_gp.subsampleIndividualsRandomly( f_prop, s_subsample_tag=str( f_prop ) )
#		#end for each proportion
#
#		s_newfile="_".join( [ mygpfile, restag, str( filecount ) ] )
#		o_gp.writeGenePopFile( s_newfile  )
#		for s_tag in o_gp.indiv_subsample_tags:
#			o_gp.writeGenePopFile( s_newfile + s_tag, s_indiv_subsample_tag=s_tag )
#		#end for each proportion
	#end for each file

	#testing new allele counts defs

	import sys

	f_genepopfile=None
	if len( sys.argv ) == 2:
		f_genepopfile=sys.argv[ 1 ]
	#end if

	MINLOCINUM=20
	MAXLOCINUM=21
	LOCILIST=[ 3,5 ]

	MININDIV=44
	MAXINDIV=45

	MINPOPNUM=24
	MAXPOPNUM=24

	opg=GenepopFileManager( f_genepopfile )

	li_pop_numbers=list(  range( MINPOPNUM, MAXPOPNUM + 1 ) )
	li_indiv_numbers=list( range( MININDIV, MAXINDIV + 1 ) )

	opg.subsamplePopulationsByList( li_pop_numbers , "popsamp" )
	opg.subsampleIndividualsByNumberList( { ipopnum : li_indiv_numbers  for ipopnum in li_pop_numbers }	, "indivsamp" )
	opg.subsampleLociByRangeAndMax( MINLOCINUM, MAXLOCINUM, "locisamp" )

	dddi_allelecounts=opg.getAlleleCounts( "popsamp", "indivsamp", "locisamp" )

	print( dddi_allelecounts )

#end if main


from django.urls import path

from . import views

app_name = "home"

urlpatterns = [
    path('', views.HomeListView.as_view(), name='index'),
    path('search/', views.SearchCarView.as_view(), name='search_car'),
    path('all_cars/', views.AllCarListView.as_view(), name='all_cars')
]


"""
Various helper utilities used internally by ncluster project, that are not explicitly tied to AWS
"""

import os
import random
import string
import sys
import time
from collections import Iterable
import shlex

from typing import Optional, Tuple

import portalocker
import paramiko

# starting value for now_micros (Aug 31, 2018), using this to make various timestamped names shorter
EPOCH_MICROS = 1535753974788163

# Whitelist of temporary settings customizable through env vars. These are work-arounds for issues that are
# don't have permanent solutions yet. Keep this small to avoid many moving parts.
env_settings = {
  'NCLUSTER_AUTHORIZED_KEYS',         # public keys used to authorize ssh access on all instances
  'NCLUSTER_AWS_FAST_ROOTDISK',       # request $1/hour high performance AWS disk
  'NCLUSTER_AWS_PLACEMENT_GROUP',     # name of placement group to use, use when adding machines to a previous launched job
  'NCLUSTER_DISABLE_PDB_HANDLER',     # don't intercept pdb exception by default
  'NCLUSTER_RUNNING_UNDER_CIRCLECI',  # special settings for non-interactive CircleCI integration test env
  #  'NCLUSTER_IMAGE',
  'NCLUSTER_SSH_USERNAME',            # used as workaround when Amazon Linux detection fails
  'NCLUSTER_ZONE',                    # zone spec for when automatic zone fails (p3dn's + spot instances)
  'NCLUSTER_AWS_FORCE_CREATE_RESOURCES',  # AWS resources are created ignoring automatic existence checks
}

# keep this here instead of aws_backend because it's used by its dependency aws_util
VALID_REGIONS = ['us-east-2',
                 'us-east-1',
                 'us-west-1',       # An error occurred (Unsupported) when calling the RunInstances operation
                 'us-west-2',
                 'ap-east-1',       # doesn't have ec2
                 'ap-south-1',      # no EFS
                 'ap-northeast-3',  # An error occurred (OptInRequired) when calling the DescribeVpcs operation
                 'ap-northeast-2',
                 'ap-southeast-1',
                 'ap-southeast-2',
                 'ap-northeast-1',
                 'ca-central-1',
                 'cn-north-1',      # account number
                 'cn-northwest-1',  # account number
                 'eu-central-1',
                 'eu-west-1',
                 'eu-west-2',
                 'eu-west-3',       # no EFS
                 'eu-north-1',      # no EFS
                 'sa-east-1',       # no EFS
                 'us-gov-east-1',   # not authorized
                 'us-gov-west-1',   # not authorized
                 ]

# print/validate custom settings
for v in os.environ:
  if v.startswith('NCLUSTER'):
    assert v in env_settings, f"Custom setting '{v}'='{os.environ[v]}' not in settings whitelist, if you" \
      f"are sure you need this setting, add it to the env_settings in {os.path.basename(__file__)}, otherwise 'unset {v}'"
    # the following are often set by default, so don't print them
    if v in {'NCLUSTER_AUTHORIZED_KEYS', 'NCLUSTER_ZONE'}:
      continue

    sys.stderr.write(f"ncluster env setting {v}={os.environ[v]}\n")

def is_iterable(k):
  return isinstance(k, Iterable)

def now_micros(absolute=False) -> int:
  """Return current micros since epoch as integer."""
  micros = int(time.time() * 1e6)
  if absolute:
    return micros
  return micros - EPOCH_MICROS

def now_millis(absolute=False) -> int:
  """Return current millis since epoch as integer."""
  millis = int(time.time() * 1e3)
  if absolute:
    return millis
  return millis - EPOCH_MICROS // 1000

def current_timestamp() -> str:
  # timestamp format from https://github.com/tensorflow/tensorflow/blob/155b45698a40a12d4fef4701275ecce07c3bb01a/tensorflow/core/platform/default/logging.cc#L80
  current_seconds = time.time()
  remainder_micros = int(1e6 * (current_seconds - int(current_seconds)))
  time_str = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(current_seconds))
  full_time_str = "%s.%06d" % (time_str, remainder_micros)
  return full_time_str

def log_error(*args, **kwargs):
  print(f"Error encountered {args} {kwargs}")

def log(*args, **kwargs):
  print(f"{args} {kwargs}")

def install_pdb_handler():
  """Automatically start pdb:
      1. CTRL+\\ breaks into pdb.
      2. pdb gets launched on exception.
  """

  import signal
  import pdb

  def handler(_signum, _frame):
    pdb.set_trace()
  signal.signal(signal.SIGQUIT, handler)

  # Drop into PDB on exception
  # from https://stackoverflow.com/questions/13174412
  def info(type_, value, tb):
   if hasattr(sys, 'ps1') or not sys.stderr.isatty():
      # we are in interactive mode or we don't have a tty-like
      # device, so we call the default hook
      sys.__excepthook__(type_, value, tb)
   else:
      import traceback
      import pdb
      # we are NOT in interactive mode, print the exception...
      traceback.print_exception(type_, value, tb)
      print()
      # ...then start the debugger in post-mortem mode.
      pdb.pm()

  sys.excepthook = info

def shell_add_echo(script):
  """Goes over each line script, adds "echo cmd" in front of each cmd.

  ls a

  becomes

  echo * ls a
  ls a
  """
  new_script = ""
  for cmd in script.split('\n'):
    cmd = cmd.strip()
    if not cmd:
      continue
    new_script += "echo \\* " + shlex.quote(cmd) + "\n"
    new_script += cmd + "\n"
  return new_script

def shell_strip_comment(cmd):
  """ hi # testing => hi"""
  if '#' in cmd:
    return cmd.split('#', 1)[0]
  else:
    return cmd

def random_id(k=5):
  """Random id to use for AWS identifiers."""
  #  https://stackoverflow.com/questions/2257441/random-string-generation-with-upper-case-letters-and-digits-in-python
  return ''.join(random.choices(string.ascii_lowercase + string.digits, k=k))

def alphanumeric_hash(s: str, size=5):
  """Short alphanumeric string derived from hash of given string"""
  import hashlib
  import base64
  hash_object = hashlib.md5(s.encode('ascii'))
  s = base64.b32encode(hash_object.digest())
  result = s[:size].decode('ascii').lower()
  return result

def reverse_taskname(name: str) -> str:
  """
  Reverses components in the name of task. Reversed convention is used for filenames since
  it groups log/scratch files of related tasks together

  0.somejob.somerun -> somerun.somejob.0
  0.somejob -> somejob.0
  somename -> somename

  Args:
    name: name of task

  """
  components = name.split('.')
  assert len(components) <= 3
  return '.'.join(components[::-1])

def is_bash_builtin(cmd):
  """Return true if command is invoking bash built-in
  """
  # from compgen -b
  bash_builtins = ['alias', 'bg', 'bind', 'alias', 'bg', 'bind', 'break',
                   'builtin', 'caller', 'cd', 'command', 'compgen', 'complete',
                   'compopt', 'continue', 'declare', 'dirs', 'disown', 'echo',
                   'enable', 'eval', 'exec', 'exit', 'export', 'false', 'fc',
                   'fg', 'getopts', 'hash', 'help', 'history', 'jobs', 'kill',
                   'let', 'local', 'logout', 'mapfile', 'popd', 'printf',
                   'pushd', 'pwd', 'read', 'readarray', 'readonly', 'return',
                   'set', 'shift', 'shopt', 'source', 'suspend', 'test',
                   'times', 'trap', 'true', 'type', 'typeset', 'ulimit',
                   'umask', 'unalias', 'unset', 'wait']
  toks = cmd.split()
  if toks and toks[0] in bash_builtins:
    return True
  return False

def is_set(name: str) -> bool:
  """Helper method to check if given property is set"""
  assert name in env_settings

  val = os.environ.get(name, '0')
  return not (val == '0')

def get_env(name: str) -> Optional[str]:
  """Helper method to retrieve custom env setting, returns None if not set"""
  assert name in env_settings
  return os.environ.get(name, None)

def set_env(name: str, value: str) -> None:
  """Helper method to set custom env setting"""
  assert name in env_settings
  os.environ[name] = value

def assert_script_in_current_directory():
  """Assert fail if current directory is different from location of the script"""

  script = sys.argv[0]
  assert os.path.abspath(os.path.dirname(script)) == os.path.abspath(
    '.'), f"Change into directory of script {script} and run again."

def validate_ncluster_job_name(name):
  assert name.count(
    '.') <= 1, "Job name has too many .'s (see ncluster design: Run/Job/Task hierarchy for  convention)"

def toseconds(dt) -> float:
  """Converts datetime object to seconds."""
  return time.mktime(dt.utctimetuple())

def wait_for_file(fn: str, max_wait_sec: int = 60,
                  check_interval: float = 1) -> bool:
    """
    Waits for file maximum of max_wait_sec. Returns True if file was detected within specified max_wait_sec
    Args:
      fn: filename
      max_wait_sec: how long to wait in seconds
      check_interval: how often to check in seconds
    Returns:
      False if waiting was was cut short by max_wait_sec limit, True otherwise
    """
    log("Waiting for file", fn)
    start_time = time.time()
    while True:
      if time.time() - start_time > max_wait_sec:
        log(f"Timeout exceeded ({max_wait_sec} sec) for {fn}")
        return False
      if not os.path.exists(fn):
        time.sleep(check_interval)
        continue
      else:
        break
    return True

# locations of default keypair
ID_RSA = os.environ['HOME'] + '/.ssh/id_rsa'
ID_RSA_PUB = ID_RSA + '.pub'

def setup_local_ssh_keys() -> str:
  """Sanity checks on local ssh keypair and regenerate it if necessary. Returns location of public keypair file"""

  if os.path.exists(ID_RSA_PUB):
    assert os.path.exists(ID_RSA), f"Public key {ID_RSA_PUB} exists but private key {ID_RSA} not found, delete {ID_RSA_PUB} and run again to regenerate pair"
    log(f"Found local keypair {ID_RSA}")
  elif os.path.exists(ID_RSA):
    if not os.path.exists(ID_RSA_PUB):
      if is_set('NCLUSTER_RUNNING_UNDER_CIRCLECI'):
        pass
      else:
        assert os.path.exists(ID_RSA_PUB), f"Private key {ID_RSA} exists but public key {ID_RSA_PUB} not found, delete {ID_RSA} and run again to regenerate pair"
    log(f"Found local keypair {ID_RSA}")
  else:
    log(f"Generating keypair {ID_RSA}")
    with portalocker.Lock(ID_RSA+'.lock', timeout=5) as _:
      os.system(f"ssh-keygen -t rsa -f {ID_RSA} -N ''")
    os.system(f'rm {ID_RSA}.lock')

  return ID_RSA_PUB

def get_authorized_keys() -> str:
  """Appends local public key to NCLUSTER_AUTHORIZED_KEYS and returns in format key1;key2;key3;
  The result can be assigned back to NCLUSTER_AUTHORIZED_KEYS env var"""

  assert os.path.exists(ID_RSA_PUB), f"{ID_RSA_PUB} not found, make sure to run 'ncluster keys'"

  current_key = open(ID_RSA_PUB).read().strip()
  auth_keys = os.environ.get('NCLUSTER_AUTHORIZED_KEYS', '')
  return auth_keys+';'+current_key+';'

def get_public_key() -> str:
  """Returns public key, creating it if needed"""

  if not os.path.exists(ID_RSA_PUB):
    print(f"{ID_RSA_PUB} not found, running sure to run setup_local_ssh_keys()")
    setup_local_ssh_keys()
    assert os.path.exists(ID_RSA_PUB)

  return open(ID_RSA_PUB).read().strip()

def exec_command(ssh: paramiko.SSHClient, command: str, bufsize=-1, timeout=None, get_pty=False, environment=None) -> Tuple[paramiko.ChannelFile, paramiko.ChannelFile, paramiko.ChannelFile, paramiko.Channel]:
    """Copy of paramiko's exec_command which also returns the channel."""

    transport: paramiko.Transport = ssh.get_transport()
    chan: paramiko.Channel = transport.open_session(timeout=timeout)
    if get_pty:
      chan.get_pty()
    chan.settimeout(timeout)
    if environment:
      chan.update_environment(environment)
    chan.exec_command(command)
    stdin: paramiko.ChannelFile = chan.makefile("wb", bufsize)
    stdout: paramiko.ChannelFile = chan.makefile("r", bufsize)
    stderr: paramiko.ChannelFile = chan.makefile_stderr("r", bufsize)
    return stdin, stdout, stderr, chan

class timeit:
  """Decorator to measure length of time spent in the block in millis and log
  it to TensorBoard."""

  def __init__(self, tag=""):
    self.tag = tag

  def __enter__(self):
    self.start = time.perf_counter()
    return self

  def __exit__(self, *args):
    self.end = time.perf_counter()
    interval_ms = 1000 * (self.end - self.start)
    print(f'timeit({self.tag}): {interval_ms})')

# no_op method/object that accept every signature
class NoOp:
  def __getattr__(self, *args):
    def no_op(*_args, **_kwargs): pass
    return no_op


class Solution:

    def __init__(self):
        self.my_stack = list()
        self.my_queue = list()

    def push_character(self, element):
        self.my_stack.append(element)

    def enqueue_character(self, element):
        self.my_queue.append(element)

    def pop_character(self):
        return self.my_stack.pop()

    def dequeue_character(self):
        return self.my_queue.pop(0)

# read the string s
s = input()
# Create the Solution class object
obj = Solution()

l = len(s)
# push/enqueue all the characters of string s to stack
for i in range(l):
    obj.push_character(s[i])
    obj.enqueue_character(s[i])

isPalindrome = True
'''
pop the top character from stack
dequeue the first character from queue
compare both the characters
'''
for i in range(l // 2):
    if obj.pop_character() != obj.dequeue_character():
        isPalindrome = False
        break
# finally print whether string s is palindrome or not.
if isPalindrome:
    print("The word, " + s + ", is a palindrome.")
else:
    print("The word, " + s + ", is not a palindrome.")


# -*- coding: utf-8 -*-
"""
Created on Tue Dec  1 22:54:14 2020

@author: Rodrigo
"""

# -*- coding: utf-8 -*-

from texttable import Texttable
import qprompt
from tabulate import tabulate
import time
start_time = time.time()

def crearGrafo():
    g = Grafo()
    while 1:

        menu = qprompt.Menu()
        menu.add("1", "Agregar nodo")
        menu.add("2", "Agregar arista")
        menu.add("3", "Bellman-Ford")
        menu.add("4", "Mostrar")
        menu.add("5", "Salir")
        choice = menu.show()

        if choice == "1":
            valor = input("Ingrese el numero del nodo: ")
            if valor not in g:
                g.agregarNodo(valor)
            else:
                print('El nodo ya existe.')
        elif choice == '2':
            origen = input("Ingrese el nodo origen: ")
            destino = input("Ingrese el nodo destino: ")
            peso = input("Ingrese el peso: ")
            if origen not in g.nodos:
                print('El nodo {} no existe'.format(origen))
            elif destino not in g.nodos:
                print('El nodo {} no existe.'.format(destino))
            else:
                if not g.existe(origen, destino):
                    g.agregarArista(origen, destino, peso)
                else:
                    print('La arista ya existe.')

        elif choice == '3':
            valor = input("Ingrese el nodo: ")
            origen = g.getNodo(valor)
            distancia = bellman_ford(g, origen)

            if distancia != 1:

                nodos = []
                for nodo in distancia:
                    nodos.append((nodo.get_key(), distancia[nodo]))
                nodos =sorted(nodos, key=lambda tup: tup[0])

                tabla = Texttable()
                row = ["Nodo"]
                row2 = ["Distancia"]
                for nodo in nodos:
                    row.append(nodo[0])
                    row2.append(nodo[1])
                tabla.add_row(row)
                tabla.add_row(row2)
                print(tabla.draw())
            else:
                print("Contiene pesos negativos")
        elif choice == "4":
            print('Nodos: ')
            for x in g:
                print(x.get_key())
            print()

            print('Aristas: ')
            for x in g:
                for destino in x.getNodoAdyacentes():
                    w = x.peso(destino)
                    print('(Origen=: {}, Destino: {}, peso: {}) '.format(x.get_key(),
                                                                 destino.get_key(), w))
            print()

        elif choice == '5':
            break

class Grafo:
    def __init__(self):
        self.nodos = {}

    def agregarNodo(self, numero):
        nodo = Nodo(numero)
        self.nodos[numero] = nodo

    def agregarArista(self, origen, destino, peso=1):
        self.nodos[origen].agregarAdyacente(self.nodos[destino], peso)

    def getNodo(self, numero):
        return self.nodos[numero]

    def __iter__(self):
        return iter(self.nodos.values())

    def __len__(self):
        return len(self.nodos)

    def existe(self, origen, destino):
        return self.nodos[origen].adyacente(self.nodos[destino])

class Nodo:
    def __init__(self, nro_nodo):
        self.nro_nodo = nro_nodo
        self.nodos_adyacentes = {}

    def get_key(self):
        return self.nro_nodo

    def agregarAdyacente(self, dest, peso):
        self.nodos_adyacentes[dest] = peso

    def getNodoAdyacentes(self):
        return self.nodos_adyacentes.keys()

    def adyacente(self, dest):
        if dest in self.nodos_adyacentes:
            return True
        else:
            return False

    def peso(self, dest):
        return self.nodos_adyacentes[dest]

def bellman_ford(grafo, origen):
    distancia = dict.fromkeys(grafo, float('inf'))
    distancia[origen] = 0

    for _ in range(len(grafo) - 1):
        for nodo in grafo:
            for adyacente in nodo.getNodoAdyacentes():
                if int(nodo.peso(adyacente)) < 0 :
                    return 1
                distancia[adyacente] = min(distancia[adyacente], distancia[nodo] + int(nodo.peso(adyacente)))

    return distancia

def main():
    crearGrafo()
    print("--- %s seconds ---" % (time.time() - start_time))

if __name__ == "__main__": main()


#!/usr/bin/env python

"""
web
----

Word Embeddings Benchmarks
"""

from __future__ import print_function

import os.path as op
import io
import sys

from   setuptools import setup, find_packages
from   setuptools.command.test import test as TestCommand

# long description
def read(*filenames, **kwargs):
    encoding = kwargs.get('encoding', 'utf-8')
    sep = kwargs.get('sep', '\n')
    buf = []
    for filename in filenames:
        with io.open(filename, encoding=encoding) as f:
            buf.append(f.read())
    return sep.join(buf)

# Get version without importing, which avoids dependency issues
MODULE_NAME = find_packages(exclude=['tests'])[0]
VERSION_PYFILE = op.join(MODULE_NAME, 'version.py')
# set __version__ variable
exec (compile(read(VERSION_PYFILE), VERSION_PYFILE, 'exec'))

# INSTALL_REQUIRES = list(parse_requirements('requirements.txt'))
# req_files = ['requirements.txt', 'pip_requirements.txt']

LICENSE = 'New BSD'

setup_dict = dict(
    name=MODULE_NAME,
    version=__version__,
    description='Word Embedding Benchmarks',

    license='New BSD',
    author='Stanislaw Jastrzebski',
    author_email='grimghil@gmail.com',
    maintainer='Stanislaw Jastrzebski',
    maintainer_email='grimghil@gmail.com',

    packages=find_packages(),

    install_requires=['numpy',
                      'scipy',
                      'scikit-learn'],

    extra_files=['CHANGES.rst', 'COPYING', 'README.rst'],

    scripts=[],

    long_description=read('README.rst', 'CHANGES.rst'),

    platforms='Linux/MacOSX',

    # https://pypi.python.org/pypi?%3Aaction=list_classifiers
    classifiers=[
        'Programming Language :: Python',
        'Development Status :: 1 - Alpha',
        'Natural Language :: English',
        'Environment :: Console',
        'Intended Audience ::  Machine Learning Research',
        'License :: OSI Approved ::' + LICENSE,
        'Operating System :: OS Independent',
        'Topic :: Software Development :: Libraries :: Python Modules',
        'Topic :: Scientific/Engineering :: Machine Learning',
        'Topic :: Scientific/Engineering :: Information Analysis',
        'Operating System :: POSIX',
        'Operating System :: Unix',
        'Operating System :: MacOS',
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.5',
    ],

    extras_require={
        'testing': ['pytest', 'pytest-cov'],
    }
)

# Python3 support keywords
if sys.version_info >= (3,):
    setup_dict['use_2to3'] = False
    setup_dict['convert_2to3_doctests'] = ['']
    setup_dict['use_2to3_fixers'] = ['']

class PyTest(TestCommand):
    user_options = [('pytest-args=', 'a', "Arguments to pass to py.test")]

    def initialize_options(self):
        TestCommand.initialize_options(self)
        self.pytest_args = []

    def finalize_options(self):
        TestCommand.finalize_options(self)
        self.test_args = []
        self.test_suite = True

    def run_tests(self):
        # import here, cause outside the eggs aren't loaded
        import pytest
        errno = pytest.main(self.pytest_args)
        sys.exit(errno)

setup_dict.update(dict(tests_require=['pytest'],
                       cmdclass={'test': PyTest}))

if __name__ == '__main__':
    setup(**setup_dict)


import random
import typing as t

from toolbox.core.consts import PromptConstants
from toolbox.core.models import Episode, Turn
from toolbox.datasets.rallio67_instruct import Rallio67InstructDataset
from toolbox.modules import BaseModule

class Rallio67InstructPDM(BaseModule):
    '''Persona Dialogue Module based on instruction following data.'''

    SYNTHETIC_PERSONAS = [
        "An artificial intelligence that can generate meaningful responses in plain English",
        "A helpful AI assistant", "Very intelligent",
        "A large language model (LLM) capable of answering user questions",
        "Highly knowledgeable and intelligent AI assistant",
        "Does their best to answer the user's questions",
        "Knows a lot, and always attempts to tell the truth"
    ]

    def generator(self) -> t.Generator[Episode, None, None]:
        for episode in Rallio67InstructDataset():
            turns: list[Turn] = [
                Turn(utterance=episode.prompt,
                     speaker=PromptConstants.USER_PREFIX,
                     human_speaker=True),
                Turn(utterance=episode.response,
                     speaker=PromptConstants.BOT_TOKEN,
                     human_speaker=False),
            ]

            persona = {
                PromptConstants.BOT_TOKEN:
                    self._generate_synthetic_persona_string()
            }

            yield Episode(turns=turns, participant_personas=persona)

    def _generate_synthetic_persona_string(self) -> str:
        selected_personas = random.sample(self.SYNTHETIC_PERSONAS, 5)
        random.shuffle(selected_personas)
        return ". ".join(selected_personas) + "."


even=0
odd=0
x=[2,4,55,67,84,88,96,1,90]
for i in range(len(x)):
    if x[i]%2==0:
        even+=1
    elif x[i]%2!=0:
        odd+=1
print("In the given set of array!! there are total")
print("Number of odd number is {0} and Number of even number is {1}".format(odd,even))


from flask import request
from flask_restful import Resource
from employeeRepository import EmployeeRepository

class Employee(Resource):

    def get(self, employee_id=None):
        employeeRepository = EmployeeRepository()
        if employee_id is None:
            return employeeRepository.find_all()
        else:
            return employeeRepository.find(str(employee_id))

    def post(self):
        employee = request.get_json()
        employeeRepository = EmployeeRepository()
        return employeeRepository.insert_new_employee(str(employee['id']), employee['name'], employee['address'])


import sys
sys.path.append(r'C:\Users\Robodia\Desktop\Python\python-raspes\ObserverPatterns')
from observer import data

if __name__ == '__main__':
    for i in data.KPI_DATA:
        if i.name == 'open':
            print('Current open ticketss: %s' % i.value)
        elif i.name == 'new':
            print('New tickets in last hour: %s' % i.value)
        elif i.name == 'closed':
            print('Tickets closed in last hour: %s' % i.value)


import uasyncio
import queue
from state import State, OutputState
from picographics import PicoGraphics, DISPLAY_PICO_DISPLAY_2, PEN_RGB332
from pimoroni import RGBLED
import gc
import my_time

class Output:
    def __init__(self) -> None:
        self._current_state = None
        self._screen_on = False
        self._screenOnHour = 0
        self._screenOnMinute = 0
        self._screenOffHour = 0
        self._screenOffMinute = 0
        self._rotated = False
        self._display = PicoGraphics(display=DISPLAY_PICO_DISPLAY_2, pen_type=PEN_RGB332)
        self._width, self.height = self._display.get_bounds()
        self._current_y_offset = 0
        self._bg_pen = self._display.create_pen(0, 0, 0)
        self._is_temp_within_range = True
        self._temp_pen_within_range = self._display.create_pen(100, 255, 100)
        self._temp_pen_outside_range = self._display.create_pen(255, 0, 100)
        self._label_pen = self._display.create_pen(255, 255, 255)
        self._value_pen = self._display.create_pen(200, 200, 255)

    async def output_loop(self, q: queue.Queue):
        led = RGBLED(6, 7, 8)
        while True:
            await self.__check_if_screen_should_be_turned_on_or_off(q)
            if not q.empty():
                output: OutputState = await q.get()
                self._current_state = output
                self.__update_screen_on_and_off_time(output)
                self.__set_rotation(output.rotated)
                self.__set_temp_within_range(output)

                self._current_y_offset = 0
                self.__set_background(output.brightness)
                self.__write_temp(output.temperature)
                self.__write_value(output.hostname, 'Name: ')
                self.__write_value(output.status, 'WiFi: ')
                self.__write_value(output.ip, 'IP: ')
                self._display.update()

            self.__set_led(led)
            await uasyncio.sleep(1.0 / 60)

    def __update_screen_on_and_off_time(self, output: OutputState):
        self._screenOnHour = output.screenOnHour
        self._screenOnMinute = output.screenOnMinute
        self._screenOffHour = output.screenOffHour
        self._screenOffMinute = output.screenOffMinute

    async def __check_if_screen_should_be_turned_on_or_off(self, q: queue.Queue):
        current_time = my_time.localtime()
        current_hour = current_time[3]
        current_minute = current_time[4]
        should_be_on = (current_hour > self._screenOnHour or (current_hour == self._screenOnHour and current_minute >= self._screenOnMinute)) and (current_hour < self._screenOffHour or (current_hour == self._screenOffHour and current_minute < self._screenOffMinute))
        if self._screen_on != should_be_on:
            self._screen_on = should_be_on
            await q.put(self._current_state)

    def __set_led(self, led):
        if not self._is_temp_within_range and my_time.localtime()[5] % 2 == 0:
            led.set_rgb(255, 0, 0)
        elif not self._current_state.connected if self._current_state is not None else True:
            led.set_rgb(255, 100, 0)
        else:
            led.set_rgb(0, 0, 0)

    def __set_rotation(self, rotated):
        if rotated != self._rotated:
            self._display = None
            gc.collect()
            rotation = 180 if rotated else 0
            self._display = PicoGraphics(display=DISPLAY_PICO_DISPLAY_2, pen_type=PEN_RGB332, rotate=rotation)
            self._rotated = rotated

    def __set_background(self, brightness):
        self._display.set_backlight(0 if not self._screen_on else brightness)
        self._display.set_pen(self._bg_pen)
        self._display.clear()

    def __write_temp(self, temp):
        pen = self._temp_pen_within_range if self._is_temp_within_range else self._temp_pen_outside_range
        def __write_f(f_offset):
            self._display.set_pen(pen)
            self._display.set_font('sans')
            thickness = 2
            self._display.set_thickness(thickness)
            scale = 1
            y_offset = 12 * scale + thickness
            self._display.text('F', f_offset, y_offset + 1, scale=scale)
        self._display.set_pen(pen)
        self._display.set_font('sans')
        thickness = 6
        self._display.set_thickness(thickness)
        scale = 4
        y_offset = 12 * scale + thickness
        text = str(f'{temp:.1f}') if temp is not None else ''
        self._display.text(text, 0, y_offset + 1, scale=scale)
        x_offset = self._display.measure_text(text, scale)
        if text is not None:
          __write_f(x_offset)
        self._current_y_offset = self._current_y_offset + 111

    def __write_value(self, value_text, label_text = None):
        self._display.set_font('bitmap8')
        thickness = 2
        self._display.set_thickness(thickness)
        scale = 2
        x_offset = 0
        if label_text is not None:
            self._display.set_pen(self._label_pen)
            self._display.text(label_text, 0, self._current_y_offset, scale=scale)
            x_offset = self._display.measure_text(label_text, scale)
        self._display.set_pen(self._value_pen)
        self._display.text(value_text, x_offset, self._current_y_offset, scale=scale)
        self._current_y_offset = self._current_y_offset + 20

    def __set_temp_within_range(self, output: OutputState):
        self._is_temp_within_range = output.temperature is None or output.min_temp <= output.temperature <= output.max_temp


import torch
import torch.nn as nn

from math import sqrt

"""VDSR Model

Very Deep Super Resolution (VDSR)

Site:
  "Accurate Image Super-Resolution Using Very Deep Convolutional Networks (2015)" (https://arxiv.org/abs/1511.04587)
"""

class VDSR(nn.Module):
  def __init__(self, c: int = 3, f: int = 3, n: int = 64, d: int = 20) -> None:
    """VDSR's Constructor

    Args:
      c (int): number of channel the input/output image.
      f (int): spatial size of region.
      n (int): number of feature map.
      d (int): number of layers. d - 1 residual layers and one output layer.

    Examples:
      >>> VSDR() # typical VDSR parameters
    """
    super().__init__()
    # Input Layer
    self.input = nn.Sequential(
      nn.Conv2d(in_channels=c, out_channels=n, kernel_size=f, padding=f//2, bias=False),
      nn.ReLU(inplace=True)
    )
    # Residual Layer
    self.residual = nn.Sequential(
      *sum([[
        nn.Conv2d(in_channels=n, out_channels=n, kernel_size=f, padding=f//2, bias=False),
        nn.ReLU(inplace=True)
      ] for _ in range(max(0, d - 1))], []),
    )
    # Output Layer
    self.output = nn.Conv2d(in_channels=n, out_channels=c, kernel_size=f, padding=f//2, bias=False)

    for m in self.modules():
      if isinstance(m, nn.Conv2d):
        m.weight.data.normal_(0, sqrt(2. / (m.kernel_size[0] * m.kernel_size[1] * m.out_channels)))
        if m.bias is not None: m.bias.data.zero_()

  def forward(self, x):
    input = x
    x = self.input(x)
    x = self.residual(x)
    x = self.output(x)
    x = torch.add(x, input)
    return x


import random
import numpy as np
import neuralNetwork as nn
import pygame
import math

#--------------------Colors-----------------------
black = (0,0,0)
white = (255,255,255)
blue = (0,0,255)
green = (0,255,0)

#---------------------sizes--------------------

display_width = 1600
display_height = 1200
wall_width= display_width/50
wall_height=display_height/6
x =  (display_width * 0.2)
y = (display_height * 0.5)

def mutation(x):
	if random.randrange(0,1)<0.1:
		offset = np.random.normal()*0.5
		newx = x + offset
		return newx
	else:
		return x

class Copter():
	def __init__(self, x, y, brain,speed =0,velocity =0, gravity=1, acceleration = 2):
		self.x = x
		self.y = y
		self.speed = speed
		self.gravity = gravity
		self.acceleration = acceleration
		self.velocity = velocity
		self.copterImg = pygame.image.load('copter.png')
		if(isinstance(brain, nn.NeuralNetwork)):
			self.brain = brain.copy()
			self.brain.mutate(mutation)
		else:
			self.brain = nn.NeuralNetwork(5,8,2)
		self.score = 0
		self.fitness = 0

	def draw(self, gameDisplay):
		gameDisplay.blit(self.copterImg, (self.x,self.y))
		#pygame.draw.rect(gameDisplay, white, [self.x, self.y, 20, 20])

	def update(self,gameDisplay, draw = False):
		self.score+=1
		self.velocity +=self.gravity
		self.y +=self.velocity
		# if draw:
		# 	self.draw(gameDisplay)

	def copy(self):
		return Copter(x,y,self.brain)

	def forward(self):
		self.x += self.speed

	def up(self):
		self.y += self.gravity

	def down(self):
		self.y -= gravity

	def think(self,wall):
		check_wall_up = wall.upper_walls[40]
		check_wall_down = wall.lower_walls[40]

		inputs = []
		inputs.append(check_wall_down.x)
		inputs.append(check_wall_up.y)
		inputs.append(check_wall_down.y)
		inputs.append(self.y)
		inputs.append(self.velocity)

		action = self.brain.predict(inputs)

		if action[1] > action[0]:
			self.velocity -= self.acceleration

class Wall_element:

	def __init__(self, x,y, width, height, color):
		self.x = x
		self.y = y
		self.width = width
		self.height = height
		self.color = color

	def draw(self,gameDisplay, xmod =0, ymod =0, wmod =0, hmod =0):
		if self.color==green:
			#pygame.draw.rect(gameDisplay, black, [self.x, 0, self.width, ],1)
			pygame.draw.rect(gameDisplay, self.color, [self.x, 0, self.width, self.y+self.height])
		else:
			#	pygame.draw.rect(gameDisplay, black, [self.x, self.y, self.width, self.height],1)
			pygame.draw.rect(gameDisplay, self.color, [self.x, self.y, self.width, display_height-self.y])

class Wall():
	def __init__(self):
		self.prev_height_top = 0
		self.prev_x = -5
		self.prev_height_bottom = display_height-wall_height
		self.upper_walls = []
		self.lower_walls = []
		self.direction = [0,0]

		for i in range(0,55):
			self.upper_walls.append(Wall_element(-wall_width+i*wall_width,0,wall_width,wall_height, green))
			self.lower_walls.append(Wall_element(-wall_width+i*wall_width,self.prev_height_bottom,wall_width,wall_height,blue))

	def update_wallgroup(self,gameDisplay,walls, draw):
		global direction
		for index in range(0,len(walls)):
			if index < len(walls)-1:
				walls[index].y = walls[index+1].y
			else:
				if self.direction[0]==1:
					walls[index].y= walls[index].y - (self.direction[1]+1)*2
					if (walls[index].color==green and walls[index].y<-150):
						self.direction = [0,0]
					self.direction[1]+=1
				else:
					walls[index].y= walls[index].y + (self.direction[1]+1)*2
					if (walls[index].color==blue and walls[index].y>display_height-20):
						self.direction = [1,0]
					self.direction[1]+=1
			if draw:
				walls[index].draw(gameDisplay)
			# if copter_x>=walls[index].x and copter_x <= (walls[index].x+walls[index].width):
			# 	if copter_y >= walls[index].y and copter_y <= (walls[index].y+walls[index].height):
			# 		return True
		return False;

	def update(self,gameDisplay,drawing=True):
		#Only update the lower walls if the upper walls did not lead to a crash
		next_direction = random.randrange(0,10)
		if next_direction <= self.direction[1]:
			self.direction = [math.floor((self.direction[0]+1)/2),0]
		self.update_wallgroup(gameDisplay, self.upper_walls, drawing)
		self.update_wallgroup(gameDisplay,self.lower_walls, drawing)

	def draw(self,gameDisplay):
		for wall in self.upper_walls:
			wall.draw(gameDisplay)
		for wall in self.lower_walls:
			wall.draw(gameDisplay)

	def crashCheck(self, copter_x,copter_y):
		for wall in self.upper_walls:
			if copter_x>=wall.x and copter_x <= (wall.x+wall.width):
				if copter_y<=wall.y+wall.height:
					return True
		for l_wall in self.lower_walls:
			if copter_x>=l_wall.x and copter_x <= (l_wall.x+l_wall.width):
				if copter_y >= l_wall.y:
					return True
		return False

def text_objects(text, font):
    textSurface = font.render(text, True, black)
    return textSurface, textSurface.get_rect()

def message_display(text):
    largeText = pygame.font.Font('freesansbold.ttf',50)
    TextSurf, TextRect = text_objects(text, largeText)
    TextRect.center = ((display_width/2),(display_height/2))
    gameDisplay.blit(TextSurf, TextRect)

    pygame.display.update()

    time.sleep(2)

def crash():
    message_display('You Crashed')

def display_score(score, gameDisplay):

    font = pygame.font.SysFont(None, 25)
    text = font.render("Score: "+str(score), True, black)
    gameDisplay.blit(text,(20,200))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Jan  8 16:18:41 2021
Sankey did not work,
I made this kind of a transition plot instead
@author: leonid
"""

import pandas as pd, os, matplotlib.pyplot as plt
#%%
os.chdir("/Users/Art/Drive/PhD/Experiments/Aging Signature/000_FINAL/12_machine_learning/selected_AS/")
os.getcwd()
df1=pd.read_csv("Best_ADAboost.tsv", index_col=0, sep="\t")
df2=pd.read_csv("Best_RNDforest.tsv", index_col=0, sep="\t")
print("index file 1", df1.index,"\nIndex file2", df2.index)
lim=50
words1=df1.index[:lim]
words2=df2.index[:lim]
#%% generate y coords for lines and arrows
Y=[]
set1=[-1]*max(len(words1),len(words2))
set2=[-1]*max(len(words1),len(words2))
for i in range(len(words1)):
    for j in range(len(words2)):
        if words1[i]==words2[j]:
            Y.append([i,j])

#%%
#generate start and stop positions
plt.rcParams['font.sans-serif'] = "Arial"
plt.rcParams['font.family'] = "sans-serif"
plt.rcParams['svg.fonttype'] = 'none'
for i in range(len(Y)):
    if min(Y[i])>=0:
        y=Y[i]
        x=[1,2]
        colr="gray"
        if y[0]>y[1]:
            colr="red"
        if y[0]<y[1]:
            colr="blue"
        plt.plot(x,y, c=colr, linewidth=2)
     #   plt.arrow(x[0],y[0],x[1]-x[0],y[1]-y[0],
      #            head_width=0.1, color=colr, linewidth=3)
     #   plt.text(x[0]-0.4,y[0],words[i], fontsize=15, color=colr)
      #  plt.text(x[1]+0.3,y[1], words[i], fontsize=15, color=colr)
plt.gca().invert_yaxis()
# words will be positioned according to their indexes
for i in range(len(words1)):
    plt.text(0.8,i,words1[i])
for i in range(len(words2)):
    plt.text(2,i,words2[i])
#plt.arrow(1,1,1,1, head_width=1)
plt.xticks([],[])
plt.yticks([],[])
plt.xlim(0,3)
plt.ylim(lim,-1)
plt.title("Best ADAboost vs RNDforest")
plt.show()
#plt.savefig("test.svg", format="svg")
# %%