<?php
/**
 * @brief		Conversion module
 * @author		<a href='https://www.invisioncommunity.com'>Invision Power Services, Inc.</a>
 * @copyright	(c) Invision Power Services, Inc.
 * @license		https://www.invisioncommunity.com/legal/standards/
 * @package		IPS Tools
 * @since		4 Sept 2013
 */

namespace IPSUtf8;

/**
 * Conversion class
 */
class Convert
{
	const VERSION_ID = '1.1.20';

	/**
	 * @brief	Data Store
	 */
	protected $_data = array();

	/**
	 * @brief	Instance
	 */
	protected static $instance = NULL;

	/**
	 * @brief	Convertable table fields
	 */
	protected static $convertCols = array( 'text', 'mediumtext', 'longtext', 'varchar', 'char', 'tinytext' );

	/**
	 * @brief	Numeric table fields
	 */
	protected static $numericCols = array( 'integer' => 11, 'int' => 11, 'smallint' => 6, 'tinyint' => 4, 'mediumint' => 8, 'bigint' => 20, 'decimal' => null, 'numeric' => null, 'float' => null, 'double' => null );

	/**
	 * @brief	Tables we need to convert
	 */
	protected static $nonUtf8Tables = NULL;

	/**
	 * @brief	Tables we need to convert
	 */
	protected static $nonUtf8Collations = NULL;

	/**
	 * @brief	Number of rows we need to convert
	 */
	protected static $rowsToConvert = NULL;

	/**
	 * @brief	Native Database object
	 */
	protected static $db = NULL;

	/**
	 * @brief	UTF-8 Database object
	 */
	protected static $utf = NULL;

	/**
	 * @brief	UTF-8 table engine
	 */
	protected static $utfTableEngine = NULL;

	/**
	 * @brief	Create don't populate
	 */
	protected static $createOnly = array( 'content_cache_posts', 'content_cache_sigs', 'sessions', 'topic_views', 'search_keywords', 'core_share_links_log', 'spider_logs', 'inline_notifications' );
	
	/**
	 * @brief	Problem Tables that can cause an error due to serialization while using "fast" mode
	 */
	protected static $problemTables = array( 'cache_store' );

	/**
	 * @brief	MySQL supported character sets
	 */
	protected static $mysqlCharSets = NULL;

	/**
	 * Get instance
	 *
	 * @return	Output
	 */
	public static function i()
	{
		if ( self::$instance === NULL )
		{
			static::$db  = \IPSUtf8\Db::i();
			if ( \defined( 'SOURCE_DB_CHARSET' ) AND SOURCE_DB_CHARSET !== NULL )
			{
				static::$db->set_charset( SOURCE_DB_CHARSET );
			}
			static::$utf = \IPSUtf8\Db::i('utf8');
			self::$instance = new self();
			self::$instance->init();
		}

		return self::$instance;
	}

	/**
	 * Get value from data store
	 *
	 * @param	mixed	$key	Key
	 * @return	mixed	Value from the datastore
	 */
	public function __get( $key )
	{
		if( isset( $this->_data[ $key ] ) )
		{
			return $this->_data[ $key ];
		}

		return NULL;
	}

	/**
	 * Set value in data store
	 *
	 * @param	mixed	$key	Key
	 * @param	mixed	$value	Value
	 * @return	void
	 */
	public function __set( $key, $value )
	{
		if( array_key_exists( $key, $this->_data ) )
		{
			$this->_data[ $key ] = $value;
		}
	}

	/**
	 * Process a batch
	 *
	 * @param	int|null	Number of rows to limit per batch
	 * @return	boolean		True (I converted some) False (I didn't)
	 */
	public function process( $limit=null, $fromError=false )
	{
		/* MB4? */
		$sessionData = \IPSUtf8\Session::i()->json;

		if ( isset( $sessionData['use_utf8mb4'] ) AND $sessionData['use_utf8mb4'] )
		{
			static::$utf->set_charset( 'utf8mb4' );
		}

		if ( \IPSUtf8\Session::i()->current_table === NULL AND \IPSUtf8\Session::i()->is_ipb )
		{
			/* Lock tasks until the end of time so they do not attempt to add / remove things during conversion */
			if( static::$db->checkForTable('task_manager') )
			{
				static::$db->update( "task_manager", array( 'task_locked' => 2147483647 ) );
			}
			else
			{
				static::$db->update( "core_tasks", array( 'next_run' => 2147483647 ) );
			}
		}

		$convertCount   = 0;
		$table          = $this->getTable();
		$convertCols    = static::getConvertableColumns( $table['name'] );
		$numericCols    = static::getNumericColumns( $table['name'] );
		$tableCharSet   = \IPSUtf8\Convert::i()->database_charset;
		$currentCharSet = \IPSUtf8\Session::i()->current_charset;

		if ( ! empty( $table['charset'] ) )
		{
			$tableCharSet = $table['charset'];

			/* Latin1 is actually cp1252, not ISO-8859-1 @link http://dev.mysql.com/doc/refman/5.0/en/charset-we-sets.html */
			/* But only if our document character set is ISO-8859-1 or UTF-8. Otherwise, we're using the slow method which needs to real charset despite what the table is set too */
			if ( $tableCharSet === 'latin1' AND \in_array( mb_strtolower( $currentCharSet ), array( 'iso-8859-1', 'utf-8' ) ) )
			{
				$currentCharSet = 'windows-1252';
			}
			
			if ( \defined( 'FORCE_CONVERT_CHARSET' ) AND FORCE_CONVERT_CHARSET !== NULL )
			{
				$currentCharSet = FORCE_CONVERT_CHARSET;
			}
		}

		if ( !\defined( 'SOURCE_DB_CHARSET' ) OR SOURCE_DB_CHARSET === NULL )
		{
			static::$db->set_charset( $tableCharSet );
		}

		/* Create table only */
		if ( IPB_LOCK and \in_array( $table['name'], static::$createOnly ) )
		{
			static::log( 'No content conversion of ' . $table['name'] . ' required' );

			$this->getNextTable( $table['name'] );

			\IPSUtf8\Session::i()->status = 'processing';
			\IPSUtf8\Session::i()->json   = $sessionData;
			\IPSUtf8\Session::i()->save();

			return true;
		}

		while( $convertCount < $limit )
		{
			$sessionData  = \IPSUtf8\Session::i()->json;

			if ( $table === null )
			{
				if ( \count( array_keys( \IPSUtf8\Session::i()->tables ) ) == \count( array_keys( \IPSUtf8\Session::i()->completed_json ) ) )
				{
					\IPSUtf8\Session::i()->status = 'completed';
					\IPSUtf8\Session::i()->save();
					return false;
				}

				return false;
			}

			/* Got any fields to convert? */
			if ( \count( $convertCols ) === 0 )
			{
				/* No data to convert, lets repopulate the easy way! */
				$this->preInsert( $table );

				static::$utf->delete( $table['name'] );
				static::$utf->insert( $table['name'], static::$db->select( '*', $table['name'] ) );

				$this->postInsert( $table );

				/* Update counts */
				$sessionData['convertedCount'] += $table['count'];
				$convertCount			       += $table['count'];

				static::log( 'No columns to convert in ' . $table['name'] . ' INSERT INTO FROM SELECT used' );

				$table        = $this->getNextTable( $table['name'] );
				$convertCols  = static::getConvertableColumns( $table['name'] );
				$numericCols  = static::getNumericColumns( $table['name'] );

				if ( $table === null or $convertCount >= $limit )
				{
					\IPSUtf8\Session::i()->status = 'processing';
					\IPSUtf8\Session::i()->json   = $sessionData;
					\IPSUtf8\Session::i()->save();

					return true;
				}
			}
			
			/* If the previous table had no columns to convert, then the table charset may be lost - reset it if it does not match */
			if ( $table !== NULL AND $tableCharSet !== $table['charset'] )
			{
				$tableCharSet = $table['charset'];
			}
			
			/* Are we reeeeeeeeally sure we know the charset? Let's check one more time */
			if ( ! $tableCharSet )
			{
				$createTable = static::$db->query( "SHOW CREATE TABLE `" . static::$db->prefix . $table['name'] . "`" )->fetch_assoc();
				
				if ( preg_match( '#\scharset=([a-z0-9]+?)(\s|$)#i', $createTable['Create Table'], $matches ) )
				{
					$tableCharSet = mb_strtolower( $matches[1] );
				}
			}

			/* Latin1 or ISO-8559-1? */
			if ( ! \in_array( $table['name'], static::$problemTables ) AND ( \in_array( $tableCharSet, $this->getMysqlCharSets() ) ) and ( \IPSUtf8\Session::i()->current_charset == 'utf-8' or \IPSUtf8\Session::i()->current_charset == 'iso-8859-1' ) AND ( !isset( $sessionData['force_conversion'] ) OR $sessionData['force_conversion'] !== 1 ) )
			{
				/* Convert the fast way! */
				$count = static::$utf->select( 'COUNT(*)', $table['name'] )->first();
				if ( $count > 0 )
				{
					/* TRUNCATE can hang so only do it if the table isn't empty (which it will be 99% of the time) */
					/* @see http://dba.stackexchange.com/questions/28055/truncate-table-statement-sometimes-hangs */
					static::log("TRUNCATE TABLE `" . static::$utf->prefix . $table['name'] . "`");

					static::$utf->query("TRUNCATE TABLE `" . static::$utf->prefix . $table['name'] . "`" );
				}

				/* Update counts */
				$sessionData['convertedCount'] += $table['count'];
				$convertCount			       += $table['count'];

				$select      	= array();
				$convertCols 	= static::getConvertableColumns( $table['name'] );
				$schematic		= static::getTableSchematic( $table['name'] );

				foreach( array_keys( $table['definition']['columns'] ) as $col )
				{
					if ( \in_array( $col, $convertCols ) )
					{
						/* Even if the current charset is set to UTF-8, if the Table Charset is latin1, we need to use this method */
						/* ... but only if the actual column isn't using utf8 */
						$columnCollation = $schematic['definition']['columns'][$col]['collation'];
						if ( UTF8_INSERT_ONLY === TRUE AND $tableCharSet == 'utf8' )
						{
							$select[] = 'CAST( `' . $col . '` AS BINARY )';
						}
						else if ( FORCE_MULTI_CONVERT === FALSE AND ( \IPSUtf8\Session::i()->current_charset != 'utf-8' OR ( $tableCharSet != 'utf8' AND ! \in_array( $columnCollation, array( 'utf8_unicode_ci', 'utf8mb4_unicode_ci' ) ) ) OR ( $tableCharSet == 'utf8' AND mb_substr( $columnCollation, 0, 5 ) == 'latin' ) ) )
						{
							if ( \IPSUtf8\Session::i()->current_charset == 'iso-8859-1' AND mb_substr( $columnCollation, 0, 5 ) == 'latin' )
							{
								/* Casting as binary truncates non-latin characters, so if we are using real Latin1, just convert */
								$select[] = 'CONVERT( `' . $col . '` USING utf8 )';
							}
							else
							{
								/* Otherwise cast to binary */
								$select[] = 'CONVERT( CAST(`' . $col . '` AS BINARY) USING utf8 )';
							}
						}
						else
						{
							/**
								This looks a little weird but bare with me. Semi-old databases (post-2.x but pre-4.x) can have UTF8 Tables and Columns, but contain Latin1 data.
								We need to try and sniff this out and apply a different type of conversion, because in this instance you cannot just extract
								as BINARY and then insert into the source. Instead, you need to convert to latin1, then to binary, then to utf8. But you can't
								do that on actual UTF8 data because it will then truncate multibyte characters so we need to compare the conversion against what
								is stored.
								Mind = blown.
							*/
							$collation = 'utf8_unicode_ci';
							$using = 'utf8';
							if ( $columnCollation == 'utf8mb4_unicode_ci' )
							{
								$collation = 'utf8mb4_unicode_ci';
								$using = 'utf8mb4';
							}
							
							$source_charset = 'latin1';
							if ( \defined( 'SOURCE_DB_CHARSET' ) AND SOURCE_DB_CHARSET !== NULL )
							{
								$source_charset = SOURCE_DB_CHARSET;
							}
							
							if ( \defined( 'MULTI_NO_CMP' ) AND MULTI_NO_CMP === TRUE )
							{
								$select[] = "CONVERT(CONVERT(CONVERT(`{$col}` USING {$source_charset}) USING BINARY) USING {$using})";
							}
							else
							{
								$select[] = 'CASE WHEN STRCMP(CONVERT(CONVERT(CONVERT(`' . $col . '` USING ' . $source_charset . ') USING BINARY) USING ' . $using . '), `' . $col . '` COLLATE ' . $collation . ') = 0 THEN CONVERT(`' . $col . '` USING BINARY) ELSE CONVERT(CONVERT(CONVERT(`' . $col . '` USING ' . $source_charset . ') USING BINARY) USING ' . $using . ') END';
							}
						}
					}
					else
					{
						$select[] = '`' . $col . '`';
					}
				}

				$this->preInsert( $table );

				$sql = "INSERT IGNORE INTO `" . static::$utf->prefix . $table['name'] . "` SELECT " . implode( ',', $select ) . " FROM `" . static::$db->prefix . $table['name'] . "`";

				static::log( $sql );

				static::$utf->query( $sql );

				$this->postInsert( $table );

				$table = $this->getNextTable( $table['name'] );

				\IPSUtf8\Session::i()->status = 'processing';
				\IPSUtf8\Session::i()->json   = $sessionData;
				\IPSUtf8\Session::i()->save();

				/* Always reset so we can test for no convertable columns above */
				return true;
			}

			/* If we have a work table, do we have a PKEY and a current row? */
			if ( \IPSUtf8\Session::i()->current_pkey )
			{
				$start = static::$utf->select('MAX(`' . \IPSUtf8\Session::i()->current_pkey . '`) as max', \IPSUtf8\Session::i()->current_table )->setKeyField('max')->first();

				$rows = static::$db->select(
							'*',
							$table['name'],
							array( \IPSUtf8\Session::i()->current_pkey . ' > ?', \intval( $start ) ),
							\IPSUtf8\Session::i()->current_pkey . ' ASC',
							( $limit ? array( 0, ( $limit - $convertCount ) ) : null )
						);
			}
			else
			{
				$start = static::$utf->select('COUNT(*) as count', \IPSUtf8\Session::i()->current_table )->setKeyField('count')->first();

				/* Fetch via offset */
				$rows = static::$db->select(
							'*',
							$table['name'],
							null,
							null,
							( $limit ? array( $start, ( $limit - $convertCount ) ) : array( $start, 18446744073709551615 ) ) # No really, this is what MySQL recommends for offset, no limit (http://dev.mysql.com/doc/refman/5.1/en/select.html#id4651990)
						);
			}

			if ( !\defined( 'FORCE_CONVERT_METHOD' ) OR FORCE_CONVERT_METHOD === NULL )
			{
				\IPSUtf8\Text\Charset::$method = 'internal';

				/* Optimise for latin character sets */
				if ( \function_exists( 'mb_convert_encoding' ) )
				{
					if ( \in_array( strtolower( $currentCharSet ), array_map( 'strtolower', mb_list_encodings() ) ) )
					{
						\IPSUtf8\Text\Charset::$method = 'mb';
					}
				}
			}

			$rowCount   = 0;
			$gotRows    = \count( $rows );
			$batch      = array();
			$batchBytes = 0;
			$max        = ( $limit ) ?: 250;

			/* If we have a text column then the data set gets large */
			if ( $limit > 50 AND static::hasTextColumn( $table['name'] ) )
			{
				$max = 50;
			}

			/* Throttle inserts for other reasons? */
			$throttle = static::throttleInserts( $table['name'] );

			if ( $throttle !== false )
			{
				$max = ( $throttle < $limit ) ? $throttle : $limit;
			}

			/* Anything fetched? */
			if ( $gotRows === 0 )
			{
				try
				{
					static::$utf->query("ALTER TABLE `" . static::$utf->prefix . "{$table['name']}` ENABLE KEYS;");
					static::log( "{$table['name']} keys enabled." );
				}
				catch( \IPSUtf8\Db\Exception $e )
				{
					static::log( static::$utf->prefix . "{$table['name']} enable keys exception caught." );
				}
				
				$table        = $this->getNextTable( $table['name'] );
				$convertCols  = static::getConvertableColumns( $table['name'] );
				$numericCols  = static::getNumericColumns( $table['name'] );

				continue;
			}

			foreach( $rows as $row )
			{
				foreach( $convertCols as $col )
				{
					if ( isset( $row[ $col ] ) AND ! empty( $row[ $col ] ) )
					{
						$currentCharSet = ( \defined( 'FORCE_CONVERT_CHARSET' ) AND FORCE_CONVERT_CHARSET != NULL ) ? FORCE_CONVERT_CHARSET : $currentCharSet;
						if ( static::isSerialized( $row[ $col ] ) AND \IPSUtf8\Text\Charset::i()->needsConverting( $row[ $col ], $currentCharSet, 'UTF-8' ) )
						{
							/* Store a copy in case it doesn't work */
							$original = $row[ $col ];
							$work = unserialize( $row[ $col ] );

							array_walk_recursive( $work, function( &$input, $key ) use ($currentCharSet)
							{
								$input = \IPSUtf8\Text\Charset::i()->convert( $input, $currentCharSet, 'UTF-8' );
							} );

							$row[ $col ] = serialize( $work );
							
							/* Did it work? */
							if ( ! @unserialize( $row[ $col ] ) )
							{
								/* Something went wrong... maybe we can figure it out */
								$work = unserialize( $original );
								array_walk_recursive( $work, function ( &$input, $key ) use ( $currentCharSet, $tableCharSet ) {
									if ( \function_exists( 'mb_detect_encoding' ) )
									{
										$encoding = mb_detect_encoding( $input );
										$input = \IPSUtf8\Text\Charset::i()->convert( $input, $encoding, 'UTF-8' );
									}
									else
									{
										/* @todo expand */
										if ( $tableCharSet == 'latin1' )
										{
											$encoding = 'ISO-8859-1';
										}
										$input = \IPSUtf8\Text\Charset::i()->convert( $input, $encoding, 'UTF-8' );
									}
								} );
								
								$row[ $col ] = serialize( $work );
							}
						}
						else
						{
							$row[ $col ] = \IPSUtf8\Text\Charset::i()->convert( $row[ $col ], $currentCharSet, 'UTF-8' );
						}
					}
				}

				/* Numeric columns */
				foreach( $numericCols as $col )
				{
					if ( isset( $row[ $col ] ) AND ! empty( $row[ $col ] ) )
					{
						if ( $row[ $col ] === null OR !\is_numeric( $row[ $col ] ) )
						{
							$row[ $col ] = 0;
						}
					}
				}

				/* Roughly add the size of this insert in bytes */
				$tmp = $row; // Use a copy so the reference in array_walk doesn't overwrite the row to be inserted

				/* Make sure it's an associative array as topic_views only has 1 column so DB driver returns an indexed array only */
				if ( \is_array( $tmp ) )
				{
					array_walk_recursive( $tmp, function( &$input, $key )
					{
						$input = htmlentities( (string) $input, ENT_QUOTES | ENT_IGNORE, 'utf-8', false );
					} );
				}

				$batchBytes += mb_strlen( @json_encode( $tmp ), '8bit');
				unset( $tmp );
				$batch[]     = $row;
				$nextBatch   = null;

				/* Give us an error margin to account for MySQL syntax */
				if ( $batchBytes > ( $this->_data['max_allowed_packet'] - ( ( $this->_data['max_allowed_packet'] / 100 ) * ( \count( $batch ) * 0.3 ) ) ) )
				{
					/* Remove last row */
					$nextBatch = array_pop( $batch );

					static::log( "Max packet hit with " . $batchBytes . 'b with ' . \count( $batch ) . ' rows' );
				}
				
				/* If we have moved the only row from $batch to $nextbatch, then process that now */
				if ( \count( $nextBatch ) and ! \count( $batch ) )
				{
					$batch     = $nextBatch;
					$nextBatch = array();
				}
				
				/* Got a batch to write? */
				if ( $nextBatch !== null OR ( \count( $batch ) === $max ) OR ( $gotRows === \count( $batch ) ) )
				{
					try
					{
						/* Optimise */
						$this->preInsert( $table );

						$insertId = static::$utf->insert( $table['name'], $batch, true, true, $table );

						if ( ! empty( static::$utf->error ) )
						{
							throw new \RuntimeException( static::$utf->error );
						}

						$this->postInsert( $table );

						/* Update counts */
						$sessionData['convertedCount'] += \count( $batch );
						$convertCount += \count( $batch );

						if ( \IPSUtf8\Session::i()->current_pkey )
						{
							\IPSUtf8\Session::i()->current_row = $insertId;
						}
						else
						{
							\IPSUtf8\Session::i()->current_row += \count( $batch );
						}

						\IPSUtf8\Session::i()->status = 'processing';
						\IPSUtf8\Session::i()->json   = $sessionData;
						\IPSUtf8\Session::i()->save();

						static::log( \count( $batch ) . " rows batch inserted using conversion method: " . \IPSUtf8\Text\Charset::$method . ', last insert ID ' . $insertId );

						$batch      = ( $nextBatch !== null ) ? array( $nextBatch ) : array();
						$batchBytes = 0;
					}
					catch( \IPSUtf8\Db\Exception $e )
					{
						$msg = $e->getMessage();

						\IPSUtf8\Session::i()->save();

						/* Trying to insert duplicate data, just let it go to the next in case there is a small overlap */
						if ( mb_stristr( $msg, 'duplicate entry' ) )
						{
							static::log( "Duplicate key failure on insert\n{$msg}\n" . var_export( $batch, true ) );
							return $this->heal( $limit, $convertCount );
						}
						else
						{
							/* Anything else we should probably halt on */
							static::log( "Exception Thrown:\n{$msg}\n" . var_export( $e, true ) );
							throw new \RuntimeException( $msg );
						}
					}
				}
			}

			/* Anything fetched? */
			if ( $gotRows === 0 )
			{
				try
				{
					static::$utf->query("ALTER TABLE `" . static::$utf->prefix . "{$table['name']}` ENABLE KEYS;");
					static::log( "{$table['name']} keys enabled." );
				}
				catch( \IPSUtf8\Db\Exception $e )
				{
					static::log( static::$utf->prefix . "{$table['name']} enable keys exception caught." );
				}
				
				$table       = $this->getNextTable( $table['name'] );
				$convertCols = static::getConvertableColumns( $table['name'] );
			}
		}

		return true;
	}


	/**
	 * Pre insert
	 *
	 * @param	array	$table	Table definition data
	 * @return void
	 */
	public function preInsert( $table )
	{
		$row = static::$utf->query( "SHOW TABLE STATUS LIKE '" . static::$utf->prefix . $table['name'] . "'" )->fetch_assoc();
		static::$utfTableEngine	= $row['Engine'];

		if ( empty( static::$utfTableEngine ) )
		{
			static::$utfTableEngine = static::$utf->defaultEngine();
		}

		if ( \strtolower( static::$utfTableEngine ) === 'myisam' )
		{
			static::log( "Pre inserts for MyISAM table " . $table['name'] );

			try
			{
				//static::$utf->query("ALTER TABLE " . static::$utf->prefix . "{$table['name']} DISABLE KEYS;");
			}
			catch( \IPSUtf8\Db\Exception $e )
			{
				static::log( static::$utf->prefix . "{$table['name']} disable keys exception caught." );
			}
		}
		else if ( \strtolower( static::$utfTableEngine ) === 'innodb' )
		{
			static::log( "Pre inserts for InnoDB table " . $table['name'] );

			/* This doesn't seem necessary
			try
			{
				static::$utf->query("SET autocommit=0");
				static::$utf->query("SET unique_checks=0");
				static::$utf->query("SET foreign_key_checks=0");
			}
			catch ( Exception $e ) { }*/
		}
	}

	/**
	 * Post insert
	 *
	 * @param	array	$table	Table definition data
	 * @return void
	 */
	public function postInsert( $table )
	{
		if ( \strtolower( static::$utfTableEngine ) === 'myisam' )
		{
			static::log( "Post inserts for MyISAM table " . $table['name'] );

			try
			{
				//static::$utf->query("ALTER TABLE " . static::$utf->prefix . "{$table['name']} ENABLE KEYS;");
			}
			catch( \IPSUtf8\Db\Exception $e )
			{
				static::log( static::$utf->prefix . "{$table['name']} enable keys exception caught." );
			}
		}
		else if ( \strtolower( static::$utfTableEngine ) === 'innodb' )
		{
			static::log( "Post inserts for InnoDB table " . $table['name'] );
			
			/* This doesn't seem necessary
			try
			{
				static::$utf->query("SET autocommit=1");
				static::$utf->query("SET unique_checks=1");
				static::$utf->query("SET foreign_key_checks=1");
			}
			catch ( Exception $e ) { }*/
		}
	}

	/**
	 * Finish the conversion
	 *
	 * @return null
	 */
	public function finish()
	{
		$charset   = 'utf8';
		$collation = 'utf8_unicode_ci';

		/* MB4? */
		$sessionData = \IPSUtf8\Session::i()->json;

		if ( isset( $sessionData['use_utf8mb4'] ) AND ! empty( $sessionData['use_utf8mb4'] ) )
		{
			static::$utf->set_charset( 'utf8mb4' );

			$charset   = 'utf8mb4';
			$collation = 'utf8mb4_unicode_ci';
		}

		/* IPB 3? */
		if ( isset( \IPSUtf8\Session::i()->tables['core_sys_conf_settings'] ) AND isset( \IPSUtf8\Session::i()->tables['cache_store'] ) )
		{
			if ( static::$utf->checkForTable('core_sys_conf_settings' ) )
			{
				static::$utf->update( 'core_sys_conf_settings', array( 'conf_value' => '' ), array( 'conf_key=?', 'gb_char_set' ) );

				$rows = static::$utf->select( '*', 'core_sys_conf_settings', array( 'conf_add_cache=?', 1 ) );

				$settings = array();
				foreach( $rows as $row ) #row your boat
				{
					$value = $row['conf_value'] != "" ?  $row['conf_value'] : $row['conf_default'];

					if ( $value == '{blank}' )
					{
						$value = '';
					}

					$settings[ $row['conf_key'] ] = $value;
				}

				static::$utf->update( 'cache_store', array( 'cs_value' => serialize( $settings ) ), array( 'cs_key=?', 'settings' ) );
			}

			/* Update Language Locales */
			if ( static::$utf->checkForTable('core_sys_lang') )
			{
				/* Store current */
				$currentLocale = setlocale( LC_ALL, '0' );

				/* Loop through languages */
				$languages	= static::$utf->select( '*', 'core_sys_lang' );
				foreach( $languages AS $language )
				{
					$locale = explode( '.', $language['lang_short'] ); # We want to update even if a charset is already set.
					foreach( array( "{$locale[0]}.UTF8", "{$locale[0]}.UTF-8", "{$locale[0]}.utf8" ) AS $test )
					{
						$verify = setlocale( LC_ALL, $test );

						if ( $verify !== FALSE )
						{
							static::$utf->update( 'core_sys_lang', array( 'lang_short' => $test ), array( 'lang_id=?', $language['lang_id'] ) );
							break;
						}
					}
				}

				foreach( explode( ";", $currentLocale ) as $locale )
				{
					$parts = explode( "=", $locale );
					if( \in_array( $parts[0], array( 'LC_ALL', 'LC_COLLATE', 'LC_CTYPE', 'LC_MONETARY', 'LC_NUMERIC', 'LC_TIME' ) ) )
					{
						setlocale( \constant( $parts[0] ), $parts[1] );
					}
				}

				/* Clear Cache so it regenerates */
				static::$utf->update( 'cache_store', array( 'cs_value' => '' ), array( 'cs_key=?', 'lang_data' ) );
			}

			/* Unlock Tasks */
			if( static::$utf->checkForTable('task_manager') )
			{
				static::$utf->update( "task_manager", array( 'task_locked' => 0 ) );
			}
			else if( static::$utf->checkForTable('core_tasks') )
			{
				static::$utf->update( "core_tasks", array( 'next_run' => time() ) );
			}
		}

		/* Change database collation */
		require( ROOT_PATH . '/conf_global.php' );

		static::$utf->query( "ALTER DATABASE `" . $INFO['sql_database'] . "` DEFAULT CHARACTER SET " . $charset . " COLLATE " . $collation );
	}

	/**
	 * Restore the original tables.
	 *
	 * @return null
	 */
	public function restoreOriginalTables()
	{
		/* Grab all tables to convert */
		$stmt = static::$db->prepare( "SHOW TABLES" );
		$stmt->execute();
		$stmt->bind_result( $name );

		$tables = array();

		while ( $stmt->fetch() === true )
		{
			$tables[] = $name;
		}

		foreach( $tables as $name )
		{
			if ( mb_substr( $name, 0, 5 ) === 'orig_' )
			{
				$plainName = mb_substr( $name, 5 );

				static::$db->query( "DROP TABLE IF EXISTS `" . $plainName . "`" );

				$rename = "RENAME TABLE `" . $name . '` TO `' . $plainName . '`';

				static::$db->query( $rename );

				static::log( $rename );
			}
		}
	}

	/**
	 * Delete the original tables.
	 *
	 * @return null
	 */
	public function deleteOriginalTables()
	{
		/* Grab all tables to convert */
		$stmt = static::$utf->prepare( "SHOW TABLES" );
		$stmt->execute();
		$stmt->bind_result( $name );

		$tables = array();

		while ( $stmt->fetch() === true )
		{
			$tables[] = $name;
		}

		foreach( $tables as $name )
		{
			if ( mb_substr( $name, 0, 5 ) === 'orig_' )
			{
				static::$db->query( "DROP TABLE IF EXISTS `" . $name . "`" );

				static::log( "DROP TABLE IF EXISTS `" . $name . "`" );
			}
		}
	}

	/**
	 * Rename the tables
	 *
	 * @return null
	 */
	public function renameTables()
	{
		/* Grab all tables to convert */
		$stmt = static::$db->prepare( "SHOW TABLES" );
		$stmt->execute();
		$stmt->bind_result( $name );

		$tables     = array();
		$origPrefix = static::$db->prefix;

		if ( mb_substr( $origPrefix, 0, 6 ) === 'x_utf_' )
		{
			$origPrefix = mb_substr( $origPrefix, 6 );
		}

		while ( $stmt->fetch() === true )
		{
			$tables[] = $name;
		}

		foreach( $tables as $name )
		{
			$tableNameNoPrefix = $name;
			$isConvertedTable  = ( mb_substr( $name, 0, 6 ) === 'x_utf_' );
			$isOrigTable	   = ( mb_substr( $name, 0, 5 ) === 'orig_' );

			if ( $isOrigTable )
			{
				continue;
			}

			if ( ! $isConvertedTable and $origPrefix )
			{
				$tableNameNoPrefix = mb_substr( $name, mb_strlen( $origPrefix ) );
			}

			/* Rename original tables */
			if ( ( ! $isConvertedTable ) and mb_substr( $name, 0, mb_strlen( $origPrefix ) ) === $origPrefix )
			{
				static::$db->query( "RENAME TABLE `" . $origPrefix . $tableNameNoPrefix . '` TO `orig_' . $origPrefix . $tableNameNoPrefix . '`' );

				static::log( "RENAME TABLE `" . $origPrefix . $tableNameNoPrefix . '` TO `orig_' . $origPrefix . $tableNameNoPrefix . '`' );
			}

			/* Grab x_utf_ prefixed tables */
			if ( $isConvertedTable )
			{
				$tableName = mb_substr( $name, mb_strlen( static::$utf->prefix ) );

				/* Rename the new one */
				if ( $tableName !== 'convert_session' AND $tableName !== 'convert_session_tables' )
				{
					static::$utf->query( "RENAME TABLE `" . $name . '` TO `' . $origPrefix . $tableName . '`' );

					static::log( "RENAME TABLE `" . $name . '` TO `' . $origPrefix . $tableName . '`' );
				}
			}
		}

		/* Update table data */
		\IPSUtf8\Session::i()->updateTableData();
	}

	/**
	 * Go through the DB and fix the collation of UTF8 tables
	 *
	 * @return null
	 */
	public function fixCollation()
	{
		$charset   = 'utf8';
		$collation = 'utf8_unicode_ci';

		/* MB4? */
		$sessionData = \IPSUtf8\Session::i()->json;

		if ( isset( $sessionData['use_utf8mb4'] ) AND ! empty( $sessionData['use_utf8mb4'] ) )
		{
			static::$utf->set_charset( 'utf8mb4' );

			$charset   = 'utf8mb4';
			$collation = 'utf8mb4_unicode_ci';
		}

		/* Change database collation */
		require( ROOT_PATH . '/conf_global.php' );

		static::$utf->query( "ALTER DATABASE `" . $INFO['sql_database'] . "` DEFAULT CHARACTER SET " . $charset . " COLLATE " . $collation );

		foreach( \IPSUtf8\Session::i()->tables as $name => $data )
		{
			if ( \in_array( $name, $this->getNonUtf8CollationTables() ) )
			{
				/* Skip if it's an _old table */
				if( mb_substr( $name, -4 ) == '_old' )
				{
					continue;
				}

				$cols      = static::getConvertableColumns( $data['name'] );
				$tableData = \IPSUtf8\Session::i()->tables[ $data['name'] ];

				if ( \count( $cols ) )
				{
					$dropIndexes    = array();
					$addIndexes     = array();
					$modify         = array();
				}

				/* Let's try an alter first - below may not be necessary */
				try
				{
					static::$utf->query( "ALTER TABLE `" . static::$db->prefix . $name . "` CONVERT TO CHARACTER SET " . $charset . " COLLATE " . $collation );
					static::$utf->query( "ALTER TABLE `" . static::$db->prefix . $name . "` DEFAULT CHARACTER SET " . $charset . " COLLATE " . $collation );
				}
				catch( \IPSUtf8\Db\Exception $e )
				{
					/* Did not work - try the long way */
					static::$utf->query( "RENAME TABLE `" . static::$db->prefix . $name . "` TO `" . static::$db->prefix . $name . "_temp`" );
					static::log( $name . " has Unique Index - temp table created from main." );
					/* Previously we used a raw create table query to create a copy, however this does not check index lengths. Leaving this for reference. */
					//static::$utf->query( "CREATE TABLE `" . static::$db->prefix . $name . "` LIKE `" . static::$db->prefix . $name . "_temp`" );
					static::$db->createTable( $this->checkTable( $tableData['definition'] ) );
					static::$utf->query( "ALTER TABLE `" . static::$db->prefix . $name . "` CONVERT TO CHARACTER SET " . $charset . " COLLATE " . $collation );
					static::$utf->query( "ALTER TABLE `" . static::$db->prefix . $name . "` DEFAULT CHARACTER SET " . $charset . " COLLATE " . $collation );
					static::log( $name . " created from temp table." );
					static::$utf->query( "REPLACE INTO `" . static::$db->prefix . $name . "` SELECT * FROM `" . static::$db->prefix . $name . "_temp`" );
					static::log( $name . " data copied to new table." );
					static::$utf->query( "DROP TABLE `" . static::$db->prefix . $name . "_temp`" );
					static::log( $name . "_temp dropped" );
				}

				if ( \count( $cols ) )
				{
					foreach( $cols as $col )
					{
						if ( isset( $tableData['definition']['columns'][ $col ] ) )
						{
							$colData = $tableData['definition']['columns'][ $col ];

							$modify[] = " MODIFY `" . $col . "` " .  $colData['type'] . ( ( \is_numeric( $colData['length'] ) AND $colData['length'] > 0 ) ? "(" . $colData['length'] . ")" : '' ) . " CHARACTER SET " . $charset . " COLLATE " . $collation;
						}
					}
				}

				if ( \count( $cols ) )
				{
					if ( \count( $modify ) )
					{
						$query = "ALTER TABLE `" . static::$db->prefix . $name . "` " . implode( ',', $modify );

						static::log( $query );
						static::$utf->query( $query );
					}

					if ( /*count( $modify ) and*/ \count( $addIndexes ) )
					{
						$query = "ALTER TABLE `" . static::$db->prefix . $name . "` " . implode( ',', $addIndexes );

						static::log( $query );
						static::$utf->query( $query );
					}
				}
			}
		}

		/* Update table data */
		\IPSUtf8\Session::i()->updateTableData();
		
		/* Update character set if we're in IP.Board */
		if ( static::$utf->checkForTable('core_sys_conf_settings' ) )
		{
			static::$utf->update( 'core_sys_conf_settings', array( 'conf_value' => '' ), array( 'conf_key=?', 'gb_char_set' ) );
		}
	}

	/**
	 * Something went wrong, so try and heal to restart progress
	 *
	 * @param	int		$limit			Process() method limit
	 * @param	int		$cycleCount		Items processed if this was in the middle of process()
	 * @return	bool|null
	 */
	public function heal( $limit=250, $cycleCount=0 )
	{
		if ( \IPSUtf8\Session::i()->current_table )
		{
			$table = \IPSUtf8\Session::i()->tables[ \IPSUtf8\Session::i()->current_table ];

			/* Table doesn't exist, so rewind back to the start of this table */
			if ( ! static::$utf->checkForTable( $table['name'] ) )
			{
				if ( static::$utf->createTable( $table['definition'] ) === false )
				{
					throw new \RuntimeException( static::$utf->error . "\n" . var_export( $table['definition'], true ) );
				}

				$completed = \IPSUtf8\Session::i()->completed_json;

				if ( isset( $completed[ \IPSUtf8\Session::i()->current_table ] ) )
				{
					unset( $completed[ \IPSUtf8\Session::i()->current_table ] );
				}

				/* Got a primary key so we can use a WHERE N > X query rather than a limit for efficiency? */
				if ( isset( $table['definition']['indexes']['PRIMARY'] ) )
				{
					$pkey = $table['definition']['indexes']['PRIMARY']['columns'][0];

					/* Is it numeric? */
					if ( mb_stristr( $table['definition']['columns'][ $pkey ]['type'], 'int' ) )
					{
						\IPSUtf8\Session::i()->current_pkey = $pkey;
					}
				}

				\IPSUtf8\Session::i()->status         = 'processing';
				\IPSUtf8\Session::i()->completed_json = $completed;
				\IPSUtf8\Session::i()->current_row    = 0;
				\IPSUtf8\Session::i()->save();

				/* Run again */
				return $this->process( ( $limit - $cycleCount ) );
			}

			if ( \IPSUtf8\Session::i()->current_pkey )
			{
				/* Get the latest row */
				$max = static::$utf->select('MAX(`' . \IPSUtf8\Session::i()->current_pkey . '`) as max', \IPSUtf8\Session::i()->current_table )->setKeyField('max')->first();
			}
			else
			{
				/* Get the count */
				$max = static::$utf->select('COUNT(*) as count', \IPSUtf8\Session::i()->current_table )->setKeyField('count')->first();
			}

			\IPSUtf8\Session::i()->status      = 'processing';
			\IPSUtf8\Session::i()->current_row = $max;
			\IPSUtf8\Session::i()->save();

			/* Run again */
			return $this->process( ( $limit - $cycleCount ), TRUE );
		}

		return null;
	}

	/**
	 * Set the current work table
	 *
	 * @return void
	 */
	public function getTable()
	{
		$table = null;

		if ( \IPSUtf8\Session::i()->current_table AND isset( \IPSUtf8\Session::i()->tables[ \IPSUtf8\Session::i()->current_table ] ) )
		{
			/* Table selected and conversion in progress */
			$table = \IPSUtf8\Session::i()->tables[ \IPSUtf8\Session::i()->current_table ];

			static::log( "Continuing with " . $table['name'] . ' (PKEY: ' . \IPSUtf8\Session::i()->current_pkey . ')' );

			if ( ! static::$utf->checkForTable( $table['name'] ) )
			{
				if ( static::$utf->createTable( $this->checkTable( $table['definition'] ) ) === false )
				{
					throw new \RuntimeException( static::$utf->error . "\n" . var_export( $table['definition'], true ) );
				}
				try
				{
					static::$utf->query("ALTER TABLE `" . static::$utf->prefix . "{$table['name']}` DISABLE KEYS;");
					static::log( "{$table['name']} keys disabled." );
				}
				catch( \IPSUtf8\Db\Exception $e )
				{
					static::log( static::$utf->prefix . "{$table['name']} disable keys exception caught." );
				}
			}

			return $table;
		}
		else
		{
			/* need to resolve an issue where if you only need to convert a few tables, these few will be named x_utf_ while others won't,
			   so for now, just convert all */
			$nonUtf8Tables = array_keys( \IPSUtf8\Session::i()->tables );//$this->getNonUtf8Tables();

			/* No table selected */
			if ( \is_array( $nonUtf8Tables ) and \count( $nonUtf8Tables ) )
			{
				if ( \is_array( \IPSUtf8\Session::i()->completed_json ) )
				{
					$diff = array_diff( $nonUtf8Tables, array_keys( \IPSUtf8\Session::i()->completed_json ) );

					if ( \count( $diff ) )
					{
						$table = \IPSUtf8\Session::i()->tables[ array_shift( $diff ) ];
					}
					else
					{
						return null;
					}
				}
				else
				{
					$table = array_shift( $nonUtf8Tables );
				}

				if ( isset( $table['name'] ) )
				{
					\IPSUtf8\Session::i()->current_table = $table['name'];

					/* Got a primary key so we can use a WHERE N > X query rather than a limit for efficiency? */
					if ( isset( $table['definition']['indexes']['PRIMARY'] ) )
					{
						$pkey = $table['definition']['indexes']['PRIMARY']['columns'][0];

						/* Is it numeric? */
						if ( mb_stristr( $table['definition']['columns'][ $pkey ]['type'], 'int' ) )
						{
							\IPSUtf8\Session::i()->current_pkey = $pkey;
						}
					}

					\IPSUtf8\Session::i()->save();

					/* Create the table for UTF8 goodness */
					static::$utf->dropTable( $table['name'], true );

					if ( static::$utf->createTable( $this->checkTable( $table['definition'] ) ) === false )
					{
						throw new \RuntimeException(  static::$utf->error . "\n" . var_export( $table['definition'], true ) );
					}

					static::log( "Created UTF8 table " . $table['name'] . ' (PKEY: ' . \IPSUtf8\Session::i()->current_pkey . ')' );

					try
					{
						static::$utf->query("ALTER TABLE `" . static::$utf->prefix . "{$table['name']}` DISABLE KEYS;");
						static::log( "{$table['name']} keys disabled." );
					}
					catch( \IPSUtf8\Db\Exception $e )
					{
						static::log( static::$utf->prefix . "{$table['name']} disable keys exception caught." );
					}

					return $table;
				}
			}
		}

		return null;
	}

	/**
	 * Attempt to fix issues with keys longer than 1000bytes
	 *
	 * @param	array	$definition		Table definition
	 * @return	array
	 */
	public function checkTable( $definition )
	{
		/* MB4? */
		$sessionData = \IPSUtf8\Session::i()->json;
		$length      = 0;
		$multiplier  = 4;
		$needsFixing = array();
		$maxLen      = 1000;

		if ( \mb_strtolower( $definition['engine'] ) === 'innodb' )
		{
			$maxLen = 767;
		}

		if ( isset( $definition['indexes'] ) )
		{
			foreach( $definition['indexes'] as $key => $index )
			{
				$thisLength = null;
				$hasText	= false;
				
				foreach( $index['columns'] as $i => $column )
				{
					$thisLength	= ( isset( $index['length'][ $i ] ) ) ? $index['length'][ $i ] : ( ( (int) $definition['columns'][ $column ]['length'] or empty( $definition['columns'][ $column ]['length'] ) ) ? $definition['columns'][ $column ]['length'] : 250 );
					
					$isText = \in_array( mb_strtolower( $definition['columns'][ $column ]['type'] ), array( 'mediumtext', 'text' ) );
					
					if ( $hasText === false and $isText === true )
					{
						$hasText = true;
					}
					
					if ( isset( $definition['columns'][ $column ] ) and ( ( ! empty( $thisLength ) or $isText ) ) )
					{
						$length += $thisLength;
					}
				}
			
				if ( ( $length * $multiplier > $maxLen ) or $hasText )
				{
					foreach( $index['columns'] as $i => $column )
					{
						$thisLength	= ( isset( $index['length'][ $i ] ) ) ? $index['length'][ $i ] : ( (int) $definition['columns'][ $column ]['length'] ? $definition['columns'][ $column ]['length'] : 250 );

						if ( isset( $definition['columns'][ $column ] ) and ( ( ! empty( $thisLength ) or \in_array( mb_strtolower( $definition['columns'][ $column ]['type'] ), array( 'mediumtext', 'text' ) ) ) ) )
						{
							/* Column name, column length, column type  */
							$needsFixing[ $key ][ $i ] = array( $column, $thisLength, $definition['columns'][ $column ]['type'] );
						}
					}
				}
						
				$length = 0;
			}
		}
		
		if ( \count( $needsFixing ) )
		{
			foreach( $needsFixing as $key => $i )
			{
				$totalLength	= 0;
				$maxChars		= $maxLen / $multiplier;

				foreach( $i as $vals )
				{
					$totalLength += $vals[1];
				}

				if ( $totalLength > $maxChars )
				{
					/* Check each column can be reduced by the amount we need reducing */
					$debt = 0;
 
					$reduceEachBy = ( ( 100 / $totalLength ) * $maxChars) / 100;
 
					/* Apply debt if we have any. We do not reduce integers */
					foreach( $i as $x => $vals )
					{
						if ( \in_array( mb_strtoupper( $vals[2] ), array_keys( static::$numericCols ) ) )
						{
							$debt += $vals[1];
						}
					}

					/* Recalculate value to multiply index sub lengths with (subtracting debt) */
					if ( $debt < $totalLength )
					{
						$reduceEachBy = ( ( 100 / ($totalLength - $debt) ) * ( $maxChars - $debt ) ) / 100;
					}
					
					foreach( $i as $x => $vals )
					{
						/* No length? */
						if ( empty( $vals[1] ) )
						{
							$vals[1] = 250;
						}

						if ( \in_array( mb_strtoupper( $vals[2] ), array_keys( static::$numericCols ) ) )
						{
							/* Preserve col len where possible but if the column length is greater than subpart allowed, NULL the length
							   otherwise MySQL will complain as you cannot use subpart on non-string column. */
							if ( $vals[1] > floor( $maxLen / $multiplier ) )
							{
								$vals[1] = NULL;
								$i[ $x ] = $vals;
							}
							
							continue;
						}

						$vals[1] = floor( $vals[1] * $reduceEachBy );	
						$i[ $x ] = $vals;
					}
				}

				foreach( $i as $x => $vals )
				{
					if ( \in_array( mb_strtolower( $vals[2] ), static::$convertCols ) AND $definition['columns'][ $definition['indexes'][ $key ]['columns'][ $x ] ]['length'] != $vals[1] )
					{
						$definition['indexes'][ $key ]['length'][ $x ] = \intval( $vals[1] );
					}
					else
					{
						$definition['indexes'][ $key ]['length'][ $x ] = NULL;
					}
				}
			}
		}

		return $definition;
	}

	/**
	 * Fetches the next table to process or false if nothing left to process
	 *
	 * @param	string	$currentTable		Name of table just processed
	 * @return	array	Array of table data
	 */
	public function getNextTable( $currentTable )
	{
		try
		{
			static::$utf->query("ALTER TABLE `" . static::$utf->prefix . "{$currentTable}` ENABLE KEYS;");
			static::log( "{$table['name']} keys enabled." );
		}
		catch( \IPSUtf8\Db\Exception $e )
		{
			static::log( static::$utf->prefix . "{$currentTable} enable keys exception caught." );
		}
		
		$completed   				= \IPSUtf8\Session::i()->completed_json;
		$completed[ $currentTable ] = $currentTable;

		\IPSUtf8\Session::i()->completed_json = $completed;
		\IPSUtf8\Session::i()->current_pkey   = null;
		\IPSUtf8\Session::i()->current_table  = null;
		\IPSUtf8\Session::i()->current_row    = 0;

		\IPSUtf8\Session::i()->save();

		return $this->getTable();
	}

	/**
	 * Init this class
	 */
	 public function init()
	 {
		/* Grab all tables to convert */
		$stmt = static::$db->query( "show variables" );

		$tables = array();

		while ( $row = $stmt->fetch_array( MYSQLI_ASSOC ) )
		{
			$key   = $row['Variable_name'];
			$value = $row['Value'];

			if ( $key == 'character_set_database' )
			{
				$this->_data['database_charset'] = strtolower( $value );
			}

			if ( $key == 'bulk_insert_buffer_size' )
			{
				$this->_data['bulk_insert_buffer_size'] = $value;
			}

			if ( $key == 'max_allowed_packet' )
			{
				$this->_data['max_allowed_packet'] = $value;
			}
		}

		if ( ! empty( $this->_data['max_allowed_packet'] ) AND ! empty( $this->_data['bulk_insert_buffer_size'] ) AND ( $this->_data['bulk_insert_buffer_size'] < $this->_data['max_allowed_packet'] ) )
		{
			$this->_data['max_allowed_packet'] = $this->_data['bulk_insert_buffer_size'];
		}

		if ( empty( $this->_data['max_allowed_packet'] ) AND empty( $this->_data['bulk_insert_buffer_size'] ) )
		{
			/* No value so use MySQL default of 1MB */
			$this->_data['max_allowed_packet'] = 1048576;
		}
	}

	/**
	 * Returns true if the table has a text or medium text column
	 *
	 * @param	string	$name		Name of table
	 * @return	boolean
	 */
	public static function hasTextColumn( $name )
	{
		$table = \IPSUtf8\Session::i()->tables[ $name ];

		foreach( $table['definition']['columns'] as $col => $val )
		{
			if ( \in_array( mb_strtolower( $val['type'] ), array( 'mediumtext', 'text' ) ) )
			{
				return true;
			}
		}

		return false;
	}


	/**
	 * Tables with lots of dynamic columns can be slow
	 *
	 * @param	string	$name		Name of table
	 * @return	boolean
	 */
	public static function throttleInserts( $name )
	{
		$table = \IPSUtf8\Session::i()->tables[ $name ];
		$total = 0;
		$var   = 0;

		foreach( $table['definition']['columns'] as $col => $val )
		{
			$total++;

			if ( \in_array( mb_strtolower( $val['type'] ), array( 'varchar' ) ) )
			{
				$var++;
			}
		}

		if ( $total > 70 )
		{
			return 50;
		}
		else if ( $total > 30 AND $var > 0 )
		{
			if ( ( $var / $total ) * 100 > 25 )
			{
				return 100;
			}
		}

		return false;
	}

	/**
	 * Returns whether the database is Invision Community already
	 *
	 * @return boolean
	 */
	public function databaseIsIPS4()
	{
		return (boolean) static::$db->checkForTable( 'core_members' );
	}
	
	/**
	 * Returns whether the DB is UTF8 already
	 *
	 * @return 	bool
	 */
	public function databaseIsUtf8()
	{
		$yeahProbably = false;

		if ( \IPSUtf8\Convert::i()->database_charset == 'utf8mb4' OR \IPSUtf8\Convert::i()->database_charset == 'utf8' OR \IPSUtf8\Session::i()->current_charset == 'utf-8' )
		{
			$yeahProbably = true;
		}

		/* Another check */
		if ( $yeahProbably === true )
		{
			/* Best check the tables, then */
			$json = \IPSUtf8\Session::i()->json;

			if ( \count( $this->getNonUtf8Tables() ) )
			{
				$yeahProbably = false;
			}
		}

		return $yeahProbably;
	}

	/**
	 * Grab the character sets from MySQL
	 *
	 * @return array
	 */
	public function getMysqlCharSets()
	{
		if ( static::$mysqlCharSets === NULL )
		{
			static::$mysqlCharSets = array( 'latin1', 'utf8' );

			$stmt = static::$db->query( "show character set" );

			while ( $row = $stmt->fetch_array( MYSQLI_ASSOC ) )
			{
				static::$mysqlCharSets[] = $row['Charset'];
			}
		}

		return static::$mysqlCharSets;
	}

	/**
	 * Return tables that need converting
	 *
	 * @param	boolean	$force	Force a recount
	 * @return array
	 */
	public function getNonUtf8Tables( $force=false )
	{
		if ( static::$nonUtf8Tables === NULL or $force === TRUE )
		{
			static::$nonUtf8Tables = array();

			/* Best check the tables, then */
			$json = \IPSUtf8\Session::i()->json;

			foreach( $json['charSets'] as $charSet => $tablesArray )
			{
				if ( ! empty( $json['force_conversion'] ) or ( $charSet != 'utf8' and $charSet != 'utf8mb4' ) )
				{
					static::$nonUtf8Tables = array_merge( static::$nonUtf8Tables, $tablesArray );
				}
			}
		}

		return static::$nonUtf8Tables;
	}

	/**
	 * Returns total rows to convert
	 *
	 * @return array
	 */
	public function getTotalRowsToConvert()
	{
		if ( static::$rowsToConvert === NULL )
		{
			static::$rowsToConvert = 0;

			foreach( array_keys( \IPSUtf8\Session::i()->tables ) as $table )
			{
				static::$rowsToConvert += \intval( \IPSUtf8\Session::i()->tables[ $table ]['count'] );
			}
		}

		return static::$rowsToConvert;
	}

	/**
	 * Returns whether the DB has the correct collation
	 *
	 * @return 	bool
	 */
	public function getNonUtf8CollationTables()
	{
		if ( static::$nonUtf8Collations === null )
		{
			static::$nonUtf8Collations = array();

			/* Best check the tables, then */
			foreach( \IPSUtf8\Session::i()->tables as $table => $definition )
			{
				/* If the table itself is not utf8_unicode_ci, don't bother check columns */
				if ( $definition['definition']['collation'] AND !\in_array( $definition['definition']['collation'], array( 'utf8_unicode_ci', 'utf8mb4_unicode_ci' ) ) )
				{
					static::$nonUtf8Collations[] = $table;
					continue;
				}

				foreach( $definition['definition']['columns'] as $name => $column )
				{
					if ( $column['collation'] and ! \in_array( $column['collation'], array( 'utf8_unicode_ci', 'utf8mb4_unicode_ci' ) ) )
					{
						static::$nonUtf8Collations[] = $table;
						break;
					}
				}
			}
		}

		return static::$nonUtf8Collations;
	}

	/**
	 * Fetch debug information
	 *
	 * @return array
	 */
	public function getDebugString()
	{
		$data           = array();
		$tableCharSet   = \IPSUtf8\Convert::i()->database_charset;
		$currentCharSet = \IPSUtf8\Session::i()->current_charset;

		$json = \IPSUtf8\Session::i()->json;
		$data[] = "IP.Board Character Set: " . $currentCharSet;
		$data[] = "Database Character Set: " . $tableCharSet;
		$data[] = "Original table prefix: " . static::$db->prefix;
		$data[] = "Converted table prefix: " . static::$utf->prefix;

		foreach( $json['charSets'] as $charSet => $tablesArray )
		{
			$data[] = \count( $tablesArray ) . ' tables are ' . $charSet;
		}

		$data[] = \count( $this->getNonUtf8CollationTables() ) . " tables have incorrect collations";
		$data[] = "Can use 'dump' method: " . var_export( $this->canDump(), true );

		return $data;
	}

	/**
	 * Detect whether this is a serialized string or not
	 *
	 * @param	string	$string	The actual string
	 * @return	boolean
	 */
	public static function isSerialized( $string )
	{
		/* If it looks nothing like a serialized string, then return it */
		if ( ! preg_match( '#^a:\d+:\{#', $string ) )
		{
			return false;
		}

		/* Now make sure it's actually a serialized string */
		return ( boolean ) @unserialize( $string );
	}

	/**
	 * Log data
	 */
	public static function log( $message )
	{
		if ( LOG )
		{
			$file  = THIS_PATH . '/tmp/log_' . date('Y-m-d') . '.cgi';
			$isNew = false;
	
			if ( ! is_file( $file ) )
			{
				$isNew = true;
			}
	
			@file_put_contents( $file, "\n" . str_repeat( '-', 48 ) . "\n" . date('r') . "\n" . $message, FILE_APPEND );
	
			if ( $isNew )
			{
				@chmod( $file, 0777 );
			}
		}
	}

	/**
	 * Return a list of numeric columns that need checking
	 *
	 * @param	string	$name	Table
	 * @return	array	Array of columns that need checking as they can contain INT
	 */
	public static function getNumericColumns( $name )
	{
		if ( ! \is_string( $name ) )
		{
			return false;
		}

		$table  = \IPSUtf8\Session::i()->tables[ $name ];
		$return = array();

		foreach( $table['definition']['columns'] as $col => $val )
		{
			if ( \in_array( mb_strtolower( $val['type'] ), static::$numericCols ) )
			{
				$return[] = $val['name'];
			}
		}

		return $return;
	}

	/**
	 * Return a list of columns that need converting
	 *
	 * @param	string	$name	Table
	 * @return	array	Array of columns that need converting as they can contain text
	 */
	public static function getConvertableColumns( $name )
	{
		if ( ! \is_string( $name ) )
		{
			return false;
		}

		$table  = \IPSUtf8\Session::i()->tables[ $name ];
		$return = array();

		foreach( $table['definition']['columns'] as $col => $val )
		{
			if ( \in_array( mb_strtolower( $val['type'] ), static::$convertCols ) )
			{
				$return[] = $val['name'];
			}
		}

		return $return;
	}

	/**
	 * Can we use the fast dump method?
	 *
	 * @return boolean
	 */
	public static function canDump()
	{
		/* Best check the tables, then */
		$json     = \IPSUtf8\Session::i()->json;
		$tablesOk = true;

		foreach( $json['charSets'] as $charSet => $tablesArray )
		{
			if ( $charSet != 'utf8' and $charSet != 'utf8mb4' and mb_substr( $charSet, 0, 5) != 'latin' and $charSet != 'windows-1252' )
			{
				$tablesOk = false;
				break;
			}
		}

		if (
			( $tablesOk ) AND
			( \IPSUtf8\Session::i()->current_charset == 'utf-8' or \IPSUtf8\Session::i()->current_charset == 'iso-8859-1' ) AND
			( ( \is_callable( 'exec' ) AND false === stripos( ini_get( 'disable_functions' ), 'exec' ) ) )
		)
		{
			try
			{
				@exec("iconv -l", $output );
			}
			catch( \ErrorException $e )
			{
				return false;
			}

			if ( ! array( $output ) or ! \count( $output ) or ( ! stristr( implode( "\n", $output ), 'latin' ) ) )
			{
				return false;
			}

			return true;
		}
		else
		{
			return false;
		}
	}
	
	/**
	 * Get Table Schematic
	 *
	 * @return	array
	 */
	protected static function getTableSchematic( $table )
	{
		return json_decode( \IPSUtf8\Db::i('utf8')->select( 'table_schema', 'convert_session_tables', array( "table_name=?", $table ) )->first(), TRUE );
	}
}
