Context

The fastboot command line is an android tool that allows to interact with an android device when in fastboot mode. I needed to switch from "fastboot mode" toward "EDL mode" (EDL stands for "Emergency Download").

If this is usually done with the command fastboot oem edl, but on the Xiaomi RedMi note 3 Pro, for some reasons, it uses a specific non-standard root command reboot-edl. This is a new occasion for showing why open-source matters, and to recompile fastboot and add the missing command.

Compiling

The following is for debian related systems. In a previous blog post I explained in detail each command for a similar hack in empathy source code.

This time, I'll give the full transcript without much explanation:

mkdir /tmp/fastboot &&
cd /tmp/fastboot &&
apt-get source android-tools-fastboot </dev/null &&
cd android-tools-*
cat <<EOF | patch -p1 &&
--- a/core/fastboot/fastboot.c
+++ b/core/fastboot/fastboot.c
@@ -284,6 +284,7 @@ void usage(void)
             "  continue                                 continue with autoboot\n"
             "  reboot                                   reboot device normally\n"
             "  reboot-bootloader                        reboot device into bootloader\n"
+            "  reboot-edl                               reboot device into EDL (RedMi Note 3 Pro)\n"
             "  help                                     show this help message\n"
             "\n"
             "options:\n"
@@ -803,6 +804,7 @@ int main(int argc, char **argv)
     int wants_wipe = 0;
     int wants_reboot = 0;
     int wants_reboot_bootloader = 0;
+    int wants_reboot_edl = 0;
     int erase_first = 1;
     void *data;
     unsigned sz;
@@ -929,6 +931,9 @@ int main(int argc, char **argv)
         } else if(!strcmp(*argv, "reboot-bootloader")) {
             wants_reboot_bootloader = 1;
             skip(1);
+        } else if(!strcmp(*argv, "reboot-edl")) {
+            wants_reboot_edl = 1;
+            skip(1);
         } else if (!strcmp(*argv, "continue")) {
             fb_queue_command("continue", "resuming boot");
             skip(1);
@@ -1009,6 +1014,8 @@ int main(int argc, char **argv)
         fb_queue_reboot();
     } else if (wants_reboot_bootloader) {
         fb_queue_command("reboot-bootloader", "rebooting into bootloader");
+    } else if (wants_reboot_edl) {
+        fb_queue_command("reboot-edl", "rebooting into EDL mode");
     }

     if (fb_queue_is_empty())
EOF

apt-get build-dep android-tools-fastboot -y </dev/null &&  ## prefix with `sudo` if needed
dpkg-buildpackage -rfakeroot -b </dev/null &&
cd .. &&
dpkg -i android-tools-fastboot*.deb  ## prefix with `sudo` if needed

You should now have a new fastboot command with the reboot-edl command.

Once your Redmi Note 3 Pro is in fastboot mode (power it up with Power and Volume-Down button pressed), you are ready to fire:

# fastboot reboot-edl
rebooting into EDL mode...
OKAY [  0.001s]
finished. total time: 0.001s

Checking that your device is now in EDL mode could be done with:

# lsusb | grep Qualcomm
Bus 001 Device 086: ID 05c6:9008 Qualcomm, Inc. Gobi Wireless Modem (QDL mode)

Happy hacking !

More info

• fastboot pre-compiled for windows on xda.

TL;DR

Fixing Windows Python 2.7 unicode issue with subprocess.Popen being unable to properly send command lines to the system.

Here is the python code fixing unicode issue

Details

# -*- coding: utf-8 -*-

from subprocess import PIPE
from subprocess_fix import Popen

def indent(text, chars=" ", first=None):
    if first:
        first_line = text.split("\n")[0]
        rest = '\n'.join(text.split("\n")[1:])
        return '\n'.join([(first + first_line).rstrip(),
                          indent(rest, chars=chars)])
    return '\n'.join([(chars + line).rstrip()
                      for line in text.split('\n')])

p = Popen(u"python reading.py ć", shell=True,
stdin=PIPE, stdout=PIPE, stderr=PIPE)
out, err = p.communicate()

if p.returncode != 0:
    print("errlvl: %s" % p.returncode)

if err:
    print("stderr:\n%s" % indent(err, " | "))
if out:
    print("stdout:\n%s" % indent(out, " | "))

# -*- coding: utf-8 -*-

import sys

import commandline_fix

print "command line received (repr): %r" % (sys.argv, )
print "command line received (str): %s" % (u" ".join(sys.argv), )

python test.py

TL;DR

More info

Traps

Here's my try to unveil the terrible truth.

Let this script separate all cases and enlighten us:

cat <<'EOF' > /tmp/test.sh
#!/bin/bash
traps_show_and_test() {
    local label="$1"
    echo; echo "TRAPS from $label:"; trap
    trap "echo modified by $label" SIGINT  ## <-- MODIFICATION
    echo "TRAPS from $label (after modification attempt):"; trap
}
export -f traps_show_and_test

traps_show_and_test main
{ traps_show_and_test subshell; }
( traps_show_and_test subparen )
bash -c 'traps_show_and_test bash_subprocess'

traps_show_and_test job & wait
{ bash -c 'traps_show_and_test subshell_job'; } & wait
( bash -c 'traps_show_and_test subparen_job' ) & wait
bash -c 'traps_show_and_test bash_job' & wait

echo
echo FINAL main traps:
trap

EOF
chmod +x /tmp/test.sh
/tmp/test.sh

Running the previous script in bash (version is 4.3.46(1)-release) gives the following:

TRAPS from main:
TRAPS from main (after modification attempt):
trap -- 'echo modified by main' SIGINT

TRAPS from subshell:
trap -- 'echo modified by main' SIGINT
TRAPS from subshell (after modification attempt):
trap -- 'echo modified by subshell' SIGINT

TRAPS from subparen:
trap -- 'echo modified by subshell' SIGINT
TRAPS from subparen (after modification attempt):
trap -- 'echo modified by subparen' SIGINT

TRAPS from bash_subprocess:
TRAPS from bash_subprocess (after modification attempt):
trap -- 'echo modified by bash_subprocess' SIGINT

TRAPS from job:
trap -- 'echo modified by subshell' SIGINT
TRAPS from job (after modification attempt):
trap -- 'echo modified by job' SIGINT

TRAPS from subshell_job:
TRAPS from subshell_job (after modification attempt):
trap -- 'echo modified by subshell_job' SIGINT

TRAPS from subparen_job:
TRAPS from subparen_job (after modification attempt):
trap -- 'echo modified by subparen_job' SIGINT

TRAPS from bash_job:
trap -- '' SIGINT
trap -- '' SIGQUIT
TRAPS from bash_job (after modification attempt):
trap -- '' SIGINT
trap -- '' SIGQUIT

FINAL main traps:
trap -- 'echo modified by subshell' SIGINT

Let's highlight important points:

• Foreground Jobs
  • subshells from {..} and (..) inherit their traps from the main scope.
  • only {..} can modify the main scope.
  • subprocesses do not inherit its trap from its parent process
  • they all can modify at least locally their traps
• Background Jobs
  • bash function inherits its traps from main scope but can't modify it (like (..))
  • processes in {..} & and (..) & do not inherit traps
  • direct background subprocess have default signed traps that are umodifiable

Bash documentation

As a reference, here are the documentation from bash related to these behaviors:

Process group id effect on background process (in Job Control section of doc):

[...] processes whose process group ID is equal to the current terminal process group ID [..] receive keyboard-generated signals such as SIGINT. These processes are said to be in the foreground. Background processes are those whose process group ID differs from the terminal's; such processes are immune to keyboard-generated signals.

Default handler for SIGINT and SIGQUIT (in Signals section of doc):

Non-builtin commands run by bash have signal handlers set to the values inherited by the shell from its parent. When job control is not in effect, asynchronous commands ignore SIGINT and SIGQUIT in addition to these inherited handlers.

and about modification of traps (in trap builtin doc):

Signals ignored upon entry to the shell cannot be trapped or reset.

Requirements

You'll need to install ecryptfs-utils which provides a few scripts that can be alleviated to achieve our goals.

Overview

As ecryptfs seems to require the usage of kernel key rings, we'll convolutedly insert the key in the kernel keyring prior the mounting.

Breakdown

Say your key is stored in the $KEY environment variable:

$ tmp_wrapped_file=/tmp/wrapped-file
$ salt=ANYTHING
$ echo -en "$KEY\n$salt" | ecryptfs-wrap-passphrase "$tmp_wrapped_file"

We have to create this 'wrapped' passphrase file before insertion in keyring:

$ echo -n "$salt" | ecryptfs-insert-wrapped-passphrase-into-keyring "$tmp_wrapped_file"

This last commands outputs your key signature (here 9f3193d42b4df62):

Inserted auth tok with sig [9f3193d42b4df62] into the user session keyring

Let's say that you store this one in $SIG and that $PATH_TO_CRYPT the path to the folder you want to mount with ecryptfs:

mount -i -t ecryptfs $PATH_TO_CRYPT $PATH_TO_CRYPT \
      -o ecryptfs_enable_filename_crypto=y,ecryptfs_passthrough=n,ecryptfs_key_bytes=16,ecryptfs_cipher=aes,ecryptfs_sig=$SIG,ecryptfs_fnek_sig=$SIG,ecryptfs_unlink_sigs=y

In the given options, you have:

• the four first are to be chosen to your convenience. Go get some general information to get the options you'd like. (These are ecryptfs_enable_filename_crypto, ecryptfs_passthrough, ecryptfs_key_bytes, ecryptfs_cipher)
• the three remaining are part of the circumvolution needed to make this work:
  • Both ecryptfs_sig, ecryptfs_fnek_sig are the ids of the key in the kernel keyring.
  • At last the ecryptfs_unlink_sigs=y will unregister the key in the kernel.

And notice the -i option to mount that bypass the "helpers" of ecryptfs-utils.

Actual script

I'm providing a ecryptfs-mount script on github. It's straight forward to use:

ecryptfs-mount PATH KEY

Don't forget to umount if finished.

Links

• https://wiki.archlinux.org/index.php/ECryptfs#Manual_setup
• https://www.kernel.org/doc/Documentation/security/keys-ecryptfs.txt

How does it work

It use docker run --entrypoint /bin/bash IMAGE on the bash script, then docker commit.

Well that's the general outline. In the detail, docker current shortcomings are making this idea much more complex than it should.

The first docker run will modify ENTRYPOINT and CMD of image, so when you commit, you break your previous values. So docker-update needs to save them, and push another commit to set them back.

docker-update shortcomings

As a quick and dirty solution, this script has some shortcomings and sharp corners. Some of these could easily repaired, and additions are welcome.

docker-update MYIMAGE <<EOF
# docker: always
apt-get update && apt-get upgrade -y
EOF

First try

Let's be naive, and we'll use read f1 f2:

$ cat /tmp/data.bin | while read f1 f2; do echo "f1: '$f1', f2: '$f2'"; done
f1: 'a1b2', f2: '3c4'
f1: '5d6n7ef9', f2: ''

You can notice that:

• NUL char where ignored for field separation
• fields where separated upon consecutive space or return, it uses value stored in IFS environment variable.
• their are only 2 lines because the \n was used to separate each record. We should use -d to specify the line delimiter.
• Note that the NUL chars are also extracted out of the data as variables don't support the NUL char.
• The \ was eaten, because read builtin parse and give it special meaning. We should use -r to avoid that.

But how should we provide the NUL delimiter to the read builtin ? knowing that you can't put NUL chars on the command line ? Hopefully I stumbled onto this blog post: http://transnum.blogspot.sg/2008/11/bashs-read-built-in-supports-0-as.html

Conclusion is that -d '' should be understood magically by bash read builtin to delimit lines with NUL characters.

Better try

Let's apply our new acquired knowledge by trying IFS=$'\0' read -d '' -r f1 f2:

$ cat /tmp/data.bin | while IFS=$'\0' read -d '' -r f1 f2; do echo "f1: '$f1', f2: '$f2'"; done
f1: 'a', f2: ''
f1: '1', f2: ''
f1: 'b', f2: ''
f1: '2 3', f2: ''
f1: 'c', f2: ''
f1: '4
  5', f2: ''
f1: 'd', f2: ''
f1: '6\n7', f2: ''
f1: 'e', f2: ''
f1: '', f2: ''
f1: 'f', f2: ''

That's much better. But notice that:

• we didn't get anything in $f2, that's normal: by specifying NUL as line delimiter (with -d '') and having NUL as field delimiter (IFS) we will be doomed to have one field per record. We will need to manage the repacking in a while loop. This doesn't sound too difficult.
• where's the final field 0A ? Hum, as there is no NUL final character in the data, read returned errlvl 1 on this last field but filled correctly the variable. A simple echo $f1 prints 9 (if you use this form: while IFS='' read -d '' -r f1 f2; do echo "f1: '$f1', f2: '$f2'"; done < /tmp/data.txt to access variables of the while).

Final Implementation ?

So knowing this, here is the final implemetation of read-0:

read-0() {
    local eof
    eof=
    while [ "$1" -a -z "$eof" ]; do
        IFS='' read -r -d '' "$1" || eof=true
        shift
    done
    test "$eof" != true -o -z "$1"
}

Final ? It surely properly works for our specification test. But what happens if EOF happens before we have fed all the variables ?:

$ echo -n "a" | while read-0 f1 f2; do echo "f1: '$f1', f2: '$f2'"; done
$

Nothing is spit out, despite the fact that we have sent a character.

This is now a specification issue: Do we want read-0 to return errorlevel 0 when it hits EOF while filing the variables ? Okay, but we said 0-sized string was a possible value... read-0 in the current specification knows it hit EOF while filling variables as your first variable can be the 0-sized string. We could make a special case, but I want to be able to distinguish a last empty element from an element.

That read-0, in the actual specification, can't do it. But we can offer a slight change in the way you build your while loop to allow that parsing.

Correct Implementation

To fill partial records, will need another specification change as current implementation will fail whenever it encounters EOF. This is an incompatible specification issue. Aside from this, we need also to take care to actually set the value of the remaining fields to the empty string. This will require to use another version of read-0:

read-0() {
    local eof
    eof=
    while [ "$1" ]; do
        IFS='' read -r -d '' -- "$1" || eof=true
        shift
    done
    test "$eof" != true
}

So this would work with read-0:

$ echo -n a | tr :  '\000' | {  eof= ; while [ -z $eof ]; do read-0 f1 f2 || eof=true; echo "f1: '$f1', f2: '$f2'"; done  }
f1: 'a', f2: ''

$ echo -n a: | tr :  '\000' | {  eof= ; while [ -z $eof ]; do read-0 f1 f2 || eof=true; echo "f1: '$f1', f2: '$f2'"; done  }
f1: 'a', f2: ''

Basically, this construct allows a last round in the loop after detecting EOF... and achieve the starting spec:

$ cat /tmp/data.bin | {  eof= ; while [ -z $eof ]; do read-0 f1 f2 || eof=true; echo "f1: '$f1', f2: '$f2'"; done  }
f1: 'a', f2: '1'
f1: 'b', f2: '2 3'
f1: 'c', f2: '4
  5'
f1: 'd', f2: '6\n7'
f1: 'e', f2: ''
f1: 'f', f2: '9
'

Trivial ?

Happy hacking.

The Griefs

Bugs were reported:

• the contact list has HUGE avatars and you can't choose to have smaller ones: what happened exactly is that the contact list skin drastically changed in a very un-wise move: each avatar took 48 pixel height, leading to un-manageably long, multi-screen tall contact list. (30 people are awaiting on this bug resolution). Worse: the option to get compact list was removed and more than 96 people are awaiting for this option to come back. A bug report was also reported on bugzilla at least twice.

  • I'm providing a fix to set avatar icon size.

• The contact list is not anymore sorted by status: yes, this means that connected and away contacts are mixed, if you have more than 10 contacts using google talk for instance, they probably be growing your contact list as away. You'll have to painfully filter by eye the ones that are really available and active.

  • I'm providing a quick fix to this also.

• You can't phone using your SIP account to your address book phone numbers.

  • Hit only 12.10, on which I've provided a fix by cherry-picking the upstream bug resolution

• No more notification of new message (launchpad report). What a failure ! I could hear message alert sound whenever I received a new message but could not get to see the incoming messages as there is no way to make the messaging window appear without the messaging indicator. Additionaly for the same reason, I could not answer to standard events as standard requests from new contacts to share our visibility status despite a pretty notification message displayed on the top right corner of my screen telling me that the request was received (and that the contact awaited an answer).

  • This one was Ubuntu's fault, and can be fixed by installing telepathy-indicator, more info following

Of course, I'm not the only one to have expressed deep concerns on theses empathy regressions.

The Cure

cd /tmp
apt-get source empathy

apt-get build-dep empathy

cd empathy-*
sed -ri 's/#define AVATAR_SIZE 48/#define AVATAR_SIZE 24/g' \
    libempathy-gtk/empathy-roster-contact.c

wget -O - "https://bug691195.bugzilla-attachments.gnome.org/attachment.cgi?id=232954&action=diff&collapsed=&context=patch&format=raw&headers=1" | patch -p 1

cat <<EOF | patch -p 1
diff --git a/libempathy-gtk/empathy-roster-view.c b/libempathy-gtk/empathy-roster-view.c
index c3ae66e..77d76d6 100644
--- a/libempathy-gtk/empathy-roster-view.c
+++ b/libempathy-gtk/empathy-roster-view.c
@@ -188,6 +188,10 @@ add_roster_contact (EmpathyRosterView *self,
   g_signal_connect (contact, "notify::alias",
       G_CALLBACK (roster_contact_changed_cb), self);

+  /* Need to resort if presence is changed */
+  g_signal_connect (contact, "notify::presence-type",
+      G_CALLBACK (roster_contact_changed_cb), self);
+
   gtk_widget_show (contact);
   gtk_container_add (GTK_CONTAINER (self), contact);

@@ -633,19 +637,76 @@ compare_roster_contacts_by_alias (EmpathyRosterContact *a,
 }

 static gint
+presence_sort_key (FolksPresenceType presence) {
+
+  switch (presence)
+    {
+      case FOLKS_PRESENCE_TYPE_UNSET:
+      case FOLKS_PRESENCE_TYPE_OFFLINE:
+      case FOLKS_PRESENCE_TYPE_UNKNOWN:
+      case FOLKS_PRESENCE_TYPE_ERROR:
+        return 0;
+
+      case FOLKS_PRESENCE_TYPE_HIDDEN:
+      case FOLKS_PRESENCE_TYPE_EXTENDED_AWAY:
+        return 1;
+
+      case FOLKS_PRESENCE_TYPE_AWAY:
+        return 2;
+
+      case FOLKS_PRESENCE_TYPE_BUSY:
+      case FOLKS_PRESENCE_TYPE_AVAILABLE:
+        return 3;
+
+      default:
+        g_warning ("Unknown FolksPresenceType: %d", presence);
+        return 0;
+  }
+
+}
+
+static gint
+compare_roster_contacts_by_presence (EmpathyRosterContact *a,
+    EmpathyRosterContact *b)
+{
+  FolksPresenceType presence_a, presence_b;
+  FolksIndividual *ind_a, *ind_b;
+  gint key_a, key_b;
+
+  ind_a = empathy_roster_contact_get_individual (a);
+  ind_b = empathy_roster_contact_get_individual (b);
+
+  presence_a = folks_presence_details_get_presence_type (
+      FOLKS_PRESENCE_DETAILS (ind_a));
+  presence_b = folks_presence_details_get_presence_type (
+      FOLKS_PRESENCE_DETAILS (ind_b));
+
+  key_a = presence_sort_key(presence_a);
+  key_b = presence_sort_key(presence_b);
+
+  return key_b - key_a;
+}
+
+static gint
 compare_roster_contacts_no_group (EmpathyRosterView *self,
     EmpathyRosterContact *a,
     EmpathyRosterContact *b)
 {
   gboolean top_a, top_b;
+  gint presence_cmp;

   top_a = contact_in_top (self, a);
   top_b = contact_in_top (self, b);

   if (top_a == top_b)
-    /* Both contacts are in the top of the roster (or not). Sort them
-     * alphabetically */
-    return compare_roster_contacts_by_alias (a, b);
+    {
+      /* Both contacts are in the top of the roster (or not). Sort them
+       * by status, then alphabetically */
+      presence_cmp = compare_roster_contacts_by_presence(a, b);
+      if (presence_cmp == 0)
+        return compare_roster_contacts_by_alias (a, b);
+      return presence_cmp;
+    }
   else if (top_a)
     return -1;
   else
@@ -676,14 +737,18 @@ compare_roster_contacts_with_groups (EmpathyRosterView *self,
     EmpathyRosterContact *b)
 {
   const gchar *group_a, *group_b;
+  gint presence_cmp;

   group_a = empathy_roster_contact_get_group (a);
   group_b = empathy_roster_contact_get_group (b);

-  if (!tp_strdiff (group_a, group_b))
+  if (!tp_strdiff (group_a, group_b)) {
     /* Same group, compare the contacts */
-    return compare_roster_contacts_by_alias (a, b);
-
+    presence_cmp = compare_roster_contacts_by_presence(a, b);
+    if (presence_cmp == 0)
+      return compare_roster_contacts_by_alias (a, b);
+    return presence_cmp;
+  }
   /* Sort by group */
   return compare_group_names (group_a, group_b);
 }
EOF

dpkg-buildpackage -rfakeroot -b

cd ..
ls -al *.deb

killall telepathy-gabble empathy-auth-client empathy-chat \
        telepathy-idle telepathy-logger telepathy-rakia  \
        telepathy-salut telepathy-haze mission-control-5 \
        empathy

sudo dpkg -i empathy_*.deb

sudo dpkg -i account-plugin-sip_*.deb

More info

• Many thanks to this how-to on rebuilding deb packages