2019.2 dev

assets/nh_files/modules/csv2sqlite.py

@@ -0,0 +1,206 @@
+#!/usr/bin/env python
+#
+# A simple Python script to convert csv files to sqlite (with type guessing)
+#
+# @author: Rufus Pollock
+# Placed in the Public Domain
+# Bug fixes by Simon Heimlicher <[email protected]> marked by `shz:'
+
+from __future__ import print_function
+
+
+import sys
+import argparse
+import csv
+import sqlite3
+import bz2
+import gzip
+from six import string_types, text_type
+
+if sys.version_info[0] > 2:
+    read_mode = 'rt'
+else:
+    read_mode = 'rU'
+
+
+def convert(filepath_or_fileobj, dbpath, table, headerspath_or_fileobj=None, compression=None, typespath_or_fileobj=None):
+    if isinstance(filepath_or_fileobj, string_types):
+        if compression is None:
+            fo = open(filepath_or_fileobj, mode=read_mode)
+        elif compression == 'bz2':
+            try:
+                fo = bz2.open(filepath_or_fileobj, mode=read_mode)
+            except AttributeError:
+                fo = bz2.BZ2File(filepath_or_fileobj, mode='r')
+        elif compression == 'gzip':
+            fo = gzip.open(filepath_or_fileobj, mode=read_mode)
+    else:
+        fo = filepath_or_fileobj
+
+    try:
+        dialect = csv.Sniffer().sniff(fo.readline())
+    except TypeError:
+        dialect = csv.Sniffer().sniff(str(fo.readline()))
+    fo.seek(0)
+
+    # get the headers
+    header_given = headerspath_or_fileobj is not None
+    if header_given:
+        if isinstance(headerspath_or_fileobj, string_types):
+            ho = open(headerspath_or_fileobj, mode=read_mode)
+        else:
+            ho = headerspath_or_fileobj
+        header_reader = csv.reader(ho, dialect)
+        headers = [header.strip() for header in next(header_reader)]
+        ho.close()
+    else:
+        reader = csv.reader(fo, dialect)
+        headers = [header.strip() for header in next(reader)]
+        fo.seek(0)
+
+    # get the types
+    if typespath_or_fileobj is not None:
+        if isinstance(typespath_or_fileobj, string_types):
+            to = open(typespath_or_fileobj, mode=read_mode)
+        else:
+            to = typespath_or_fileobj
+        type_reader = csv.reader(to, dialect)
+        types = [_type.strip() for _type in next(type_reader)]
+        to.close()
+    else:
+        # guess types
+        type_reader = csv.reader(fo, dialect)
+        if not header_given: next(type_reader)
+        types = _guess_types(type_reader, len(headers))
+        fo.seek(0)
+
+    # now load data
+    _columns = ','.join(
+        ['"%s" %s' % (header, _type) for (header,_type) in zip(headers, types)]
+    )
+
+    reader = csv.reader(fo, dialect)
+    if not header_given: # Skip the header
+        next(reader)
+
+    conn = sqlite3.connect(dbpath)
+    # shz: fix error with non-ASCII input
+    conn.text_factory = str
+    c = conn.cursor()
+
+    try:
+        create_query = 'CREATE TABLE %s (%s)' % (table, _columns)
+        c.execute(create_query)
+    except:
+        pass
+
+    _insert_tmpl = 'INSERT INTO %s VALUES (%s)' % (table,
+                                                   ','.join(['?']*len(headers)))
+
+    line = 0
+    for row in reader:
+        line += 1
+        if len(row) == 0:
+            continue
+        # we need to take out commas from int and floats for sqlite to
+        # recognize them properly ...
+        try:
+            row = [
+                None if x == ''
+                else float(x.replace(',', '')) if y == 'real'
+                else int(x) if y == 'integer'
+                else x for (x,y) in zip(row, types) ]
+            c.execute(_insert_tmpl, row)
+        except ValueError as e:
+            print("Unable to convert value '%s' to type '%s' on line %d" % (x, y, line), file=sys.stderr)
+        except Exception as e:
+            print("Error on line %d: %s" % (line, e), file=sys.stderr)
+
+
+    conn.commit()
+    c.close()
+
+def _guess_types(reader, number_of_columns, max_sample_size=100):
+    '''Guess column types (as for SQLite) of CSV.
+    :param fileobj: read-only file object for a CSV file.
+    '''
+    # we default to text for each field
+    types = ['text'] * number_of_columns
+    # order matters
+    # (order in form of type you want used in case of tie to be last)
+    options = [
+        ('text', text_type),
+        ('real', float),
+        ('integer', int)
+        # 'date',
+    ]
+    # for each column a set of bins for each type counting successful casts
+    perresult = {
+        'integer': 0,
+        'real': 0,
+        'text': 0
+    }
+
+    results = [ dict(perresult) for x in range(number_of_columns) ]
+    sample_counts = [ 0 for x in range(number_of_columns) ]
+
+    for row_index,row in enumerate(reader):
+        for column,cell in enumerate(row):
+            cell = cell.strip()
+            if len(cell) == 0:
+                continue
+
+            # replace ',' with '' to improve cast accuracy for ints and floats
+            if(cell.count(',') > 0):
+                cell = cell.replace(',', '')
+                if(cell.count('E') == 0):
+                    cell = cell + "E0"
+
+            for data_type,cast in options:
+                try:
+                    cast(cell)
+                    results[column][data_type] += 1
+                    sample_counts[column] += 1
+                except ValueError:
+                    pass
+
+        have_max_samples = True
+        for column,cell in enumerate(row):
+            if sample_counts[column] < max_sample_size:
+                have_max_samples = False
+
+        if have_max_samples:
+            break
+
+    for column,colresult in enumerate(results):
+        for _type, _ in options:
+            if colresult[_type] > 0 and colresult[_type] >= colresult[types[column]]:
+                types[column] = _type
+
+    return types
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='''
+Convert a CSV file to a table in a SQLite database.
+The database is created if it does not yet exist.
+''')
+    parser.add_argument('csv_file', type=str, help='Input CSV file path')
+    parser.add_argument('sqlite_db_file', type=str, help='Output SQLite file')
+    parser.add_argument('table_name', type=str, nargs='?', help='Name of table to write to in SQLite file', default='data')
+    parser.add_argument('--headers', type=str, nargs='?', help='Headers are read from this file, if provided.', default=None)
+    parser.add_argument('--types', type=list, nargs='?', help='Types are read from this file, if provided.', default=None)
+
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument('--bz2', help='Input csv file is compressed using bzip2.', action='store_true')
+    group.add_argument('--gzip', help='Input csv file is compressed using gzip.', action='store_true')
+
+    args = parser.parse_args()
+
+    compression = None
+    if args.bz2:
+        compression = 'bz2'
+    elif args.gzip:
+        compression = 'gzip'
+
+    convert(args.csv_file, args.sqlite_db_file, args.table_name, args.headers, compression, args.types)

assets/scripts/bootkali

@@ -17,7 +17,6 @@ SCRIPT_PATH=$(readlink -f $0)
 if [ $# -eq 0 ]; then
 	$busybox chroot $mnt /bin/bash -c "if [ ! -f $HOME/.hushlogin ]; then touch $HOME/.hushlogin; fi; /bin/login -f root"
 else
-
 	#APACHE
 
 	if [ "$1" == "apache" ] && [ "$2" == "start" ]; then
@@ -154,7 +153,7 @@ else
 		if [ ! -x $mnt/usr/bin/hid-keyboard ]; then
 		    bklog "[!] No hid-keyboard executable is found in kali, now trying to compile it."
 		    if [ -e /sdcard/nh_files/modules/hid-keyboard.c ]; then
-		        $busybox chroot $mnt /usr/bin/sudo apt install gcc binutils
+		        $busybox chroot $mnt /usr/bin/sudo apt install -y python gcc binutils
 		        $busybox chroot $mnt /usr/bin/sudo gcc /sdcard/nh_files/modules/hid-keyboard.c -o /usr/bin/hid-keyboard
 				$busybox chroot $mnt /usr/bin/sudo chmod 755 /usr/bin/hid-keyboard
 			fi

assets/scripts/bootkali_init

@@ -18,8 +18,10 @@ else
 	exit 1
 fi
 
+
 ## No longer needed as SELinux is set to permissive mode via "10setenforce" init script.
-#su -c setenforce 0
+#su -c 'setenforce 0'
+
 
 ######### CHECK FOR ROOT #########
 f_checkforroot(){
@@ -40,6 +42,16 @@ nhsys=/data/local/nhsystem
 # remove chroot.  So stop here if removed or else
 # chroot is never uninstalled.
 
+####### SYSFILE CREATION #######
+# Responsible for creating files which are not found in Android,
+# but are needed in Kali.
+
+if [ ! -e "/dev/stdin" -o ! -e "/dev/stdout" -o ! -e "/dev/stderr" ]; then
+	[ -e "/dev/stdin" ] || ln -s /proc/self/fd/0 /dev/stdin
+	[ -e "/dev/stdout" ] || ln -s /proc/self/fd/1 /dev/stdout
+	[ -e "/dev/stderr" ] || ln -s /proc/self/fd/2 /dev/stderr
+fi
+
 
 ######### MOUNT #########
 mount_sdcard() {

assets/scripts/chroot_backup

@@ -0,0 +1,23 @@
+#!/system/bin/sh
+
+# Script to backup your kali chroot folder as a tar.gz file.
+
+######### IMPORT BOOTKALI INITIALIZATION #########
+SCRIPT_PATH=$(readlink -f $0)
+. ${SCRIPT_PATH%/*}/bootkali_log
+
+[ ! "`which busybox`" ] && bklog "[-] No busybox is installed, if you did have it installed, please symlink it to /system/bin"
+
+case $# in
+2)
+	[ ! -d $1 ] && bklog "[-] $1 not found, please check again." && exit 1;
+	[ -e $2 ] && bklog "[-] Found existing $2, please rename it first." && exit 1;
+	target_dir=`echo "$1" | sed "s/\(\/\+\)$//g" | awk -F/ '{print $NF}'`
+	bklog "[!] Running killkali to make sure no kali service or process is being run." && killkali
+	bklog "[!] Creating chroot backup $target_dir to $2" && cd $1/../ && busybox tar czf $2 $target_dir
+	;;
+*)
+	echo "[!] Usage: chroot_backup <full_path_of_your_chroot_folder> <full_path_with_desired_name.tar.gz>"
+	echo "[!] For example: chroot_backup /data/local/nhsystem/kali_armhf /sdcard/kalifs-backup.tar.gz"
+	;;
+esac

assets/scripts/chroot_restore

@@ -0,0 +1,22 @@
+#!/system/bin/sh
+
+# Script to restore your kali chroot folder.
+
+######### IMPORT BOOTKALI INITIALIZATION #########
+SCRIPT_PATH=$(readlink -f $0)
+. ${SCRIPT_PATH%/*}/bootkali_log
+
+[ ! "`which busybox`" ] && bklog "[-] No busybox is installed, if you did have it installed, please symlink it to /system/bin"
+
+case $# in
+2)
+	[ ! -e $1 ] && bklog "[-] $1 not found, please check again." && exit 1;
+	[ ! -d $2 ] && bklog "[-] $2 not existed, please check again." && exit 1;
+	bklog "[!] Restoring the chroot backup from $1 to $2"
+	busybox tar xzf $1 -C `echo "$2" | sed "s/\(\/\+\)$//g"`/
+	;;
+*)
+	echo "[!] Usage: chroot_restore <full_path_of_your_.tar.gz> </data/local/nhsystem/>"
+	echo "[!] For example: chroot_restore /sdcard/kalifs-backup.tar.gz /data/local/nhsystem/"
+	;;
+esac

assets/scripts/killkali

@@ -29,6 +29,9 @@ unset LD_PRELOAD
 
 $busybox sysctl -w kernel.shmmax=134217728
 
+bklog "[!] Shutting down dbus"
+$busybox chroot $mnt /etc/init.d/dbus stop
+
 bklog "[!] Shutting down SSH Server"
 $busybox chmod 666 /dev/null
 

build.gradle

@@ -49,7 +49,7 @@ android {
         minSdkVersion 17
         targetSdkVersion 28
         versionCode 22
-        versionName "2019.2-rc4"
+        versionName "2019.2-rc5"
     }
 
     sourceSets {

res/layout/metapackagechooser.xml

@@ -22,12 +22,6 @@
             android:id="@+id/metapackageLinearLayout"
             android:padding="16dp">
 
-            <CheckBox
-                android:checked="true"
-                android:layout_width="match_parent"
-                android:layout_height="wrap_content"
-                android:text="FixedUpdatingToLatestKali" />
-
             <CheckBox
                 android:checked="true"
                 android:layout_width="match_parent"

src/com/offsec/nethunter/ChrootManagerFragment.java

@@ -105,7 +105,7 @@
     Generated by tools/pin.py
     @ https://github.com/moxie0/AndroidPinning
     */
-    private final String[] pins = new String[]{"56a771798c059a990c948db4bea5cb5965d875be"};
+    private final String[] pins = new String[]{"d2599df9433782d82d438bd78d1573e6b80dfce1"};
     private String SHA512_MINIMAL;
     private String SHA512_FULL;
     private String SHA512;
@@ -292,7 +292,7 @@ private void downloadOrSdcard() {
 
         AlertDialog.Builder adb = new AlertDialog.Builder(getActivity());
         adb.setTitle("Select chroot install mode:")
-                .setMessage("Download is the prefered mode. Get the latest chroot from the offsec servers.\n\nYou can place a custom\nkalifs-[minimal|full].tar.xz in /sdcard\nand skip the download.\n\nAlso, You can place a back up kalifs-backup.tar.gz in /sdcard to restore your backup chroot.")
+                .setMessage("Download is the preferred mode. Get the latest chroot from the offsec servers.\n\nYou can place a custom\nkalifs-[minimal|full].tar.gz in /sdcard\nand skip the download.\n\nAlso, You can place a back up kalifs-backup.tar.gz in /sdcard to restore your backup chroot.")
                 .setNegativeButton("Restore from SdCard", new DialogInterface.OnClickListener() {
                     @Override
                     public void onClick(DialogInterface dialog, int which) {
@@ -405,11 +405,11 @@ private void installAndUpgrade(String packages) {
         try {
             Intent intent = new Intent("com.offsec.nhterm.RUN_SCRIPT_NH");
             intent.addCategory(Intent.CATEGORY_DEFAULT);
-            intent.putExtra("com.offsec.nhterm.iInitialCommand", nh.makeTermTitle("Updating") + "echo \"Updating gpg key..\" && wget -q -O - https://archive.kali.org/archive-key.asc | apt-key add && apt-get update && apt-get install " + packages + " -y && apt-get dist-upgrade -y && echo \"\nKali Linux Nethunter setup is complete.\nEnjoy. (You can close the terminal now)\n\"");
+            intent.putExtra("com.offsec.nhterm.iInitialCommand", nh.makeTermTitle("Updating") + "echo \"Updating gpg key..\" && wget -q -O - https://archive.kali.org/archive-key.asc | apt-key add && apt update && apt install " + packages + " -y && apt full-upgrade -y && echo \"\nKali Linux NetHunter setup is complete.\nEnjoy. (You can close the terminal now)\n\"");
             if (packages.equals("FixedUpdatingToLatestKali")) {
-                intent.putExtra("com.offsec.nhterm.iInitialCommand", nh.makeTermTitle("Updating") + "echo \"Updating gpg key..\" && wget -q -O - https://archive.kali.org/archive-key.asc | apt-key add && apt-get update && apt-get install kali-linux-all -y && apt-get dist-upgrade -y && echo \"\nKali Linux Nethunter setup is complete.\nEnjoy. (You can close the terminal now)\n\"");
+                intent.putExtra("com.offsec.nhterm.iInitialCommand", nh.makeTermTitle("Updating") + "echo \"Updating gpg key..\" && wget -q -O - https://archive.kali.org/archive-key.asc | apt-key add && apt update && apt install kali-linux-all -y && apt full-upgrade -y && echo \"\nKali Linux NetHunter setup is complete.\nEnjoy. (You can close the terminal now)\n\"");
             } else {
-                intent.putExtra("com.offsec.nhterm.iInitialCommand", nh.makeTermTitle("Updating") + "apt-get update && apt-get install " + packages + " && echo \"\nUpgrade completed.\nEnjoy. (You can close the terminal now)\n\"");
+                intent.putExtra("com.offsec.nhterm.iInitialCommand", nh.makeTermTitle("Updating") + "apt update && apt install " + packages + " -y && echo \"\nUpgrade completed.\nEnjoy. (You can close the terminal now)\n\"");
             }
             Log.d("PACKS:", "PACKS:" + packages);
             startActivity(intent);
@@ -659,8 +659,10 @@ protected Boolean doInBackground(Void... Void) {
                 }
                 // Decompress, extract, and deploy the .tar.xz to the chroot destination in one step
                 publishProgress(getActivity().getString(R.string.extract_chroot));
-                if ( zipFilePath.contains("tar.gz")) {
-                    x.RunAsRootOutput(nh.whichBusybox() + " tar -xzf '" + zipFilePath + "' -C '" + nh.NH_SYSTEM_PATH + "'");
+                Log.d(TAG, "Restoring kali chroot from " + zipFilePath + " to " + nh.NH_SYSTEM_PATH);
+
+                if (zipFilePath.contains("tar.gz")) {
+                    x.RunAsRootOutput(nh.APP_SCRIPTS_PATH + "/chroot_restore " + zipFilePath + " " + nh.NH_SYSTEM_PATH);
                 } else {
                     x.RunAsRootOutput(nh.whichBusybox() + " tar -xJf '" + zipFilePath + "' -C '" + nh.NH_SYSTEM_PATH + "'");
                 }