Merge branch 'feature/upstream' into develop

3 files changed, 177 insertions, 161 deletions

diff --git a/plugins/common/OAuth1Support.vala b/plugins/common/OAuth1Support.vala
index e5a8545..05ba34f 100644
--- a/plugins/common/OAuth1Support.vala
+++ b/plugins/common/OAuth1Support.vala
@@ -6,7 +6,6 @@
  */
 
 namespace Publishing.RESTSupport.OAuth1 {
-    internal const string ENCODE_RFC_3986_EXTRA = "!*'();:@&=+$,/?%#[] \\";
 
     public class Session : Publishing.RESTSupport.Session {
         private string? request_phase_token = null;
@@ -62,7 +61,7 @@ namespace Publishing.RESTSupport.OAuth1 {
             Publishing.RESTSupport.Argument[] sorted_args =
                 Publishing.RESTSupport.Argument.sort(base_string_arguments);
 
-            var arguments_string = Argument.serialize_list(sorted_args);
+            var arguments_string = Argument.serialize_for_sbs(sorted_args);
 
             string? signing_key = null;
             if (access_phase_token_secret != null) {
@@ -80,9 +79,9 @@ namespace Publishing.RESTSupport.OAuth1 {
                 signing_key = consumer_secret + "&";
             }
 
-            string signature_base_string = http_method + "&" + Soup.URI.encode(
-                    txn.get_endpoint_url(), ENCODE_RFC_3986_EXTRA) + "&" +
-                Soup.URI.encode(arguments_string, ENCODE_RFC_3986_EXTRA);
+            string signature_base_string = http_method + "&" + GLib.Uri.escape_string(
+                    txn.get_endpoint_url()) + "&" +
+                GLib.Uri.escape_string (arguments_string);
 
             debug("signature base string = '%s'", signature_base_string);
 
@@ -90,7 +89,7 @@ namespace Publishing.RESTSupport.OAuth1 {
 
             // compute the signature
             string signature = RESTSupport.hmac_sha1(signing_key, signature_base_string);
-            signature = Soup.URI.encode(signature, ENCODE_RFC_3986_EXTRA);
+            signature = GLib.Uri.escape_string(signature);
 
             debug("signature = '%s'", signature);
 
@@ -111,11 +110,8 @@ namespace Publishing.RESTSupport.OAuth1 {
         }
 
         public string get_oauth_nonce() {
-            TimeVal currtime = TimeVal();
-            currtime.get_current_time();
-
-            return Checksum.compute_for_string(ChecksumType.MD5, currtime.tv_sec.to_string() +
-                    currtime.tv_usec.to_string());
+            return Checksum.compute_for_string(ChecksumType.MD5,
+                                               GLib.get_real_time().to_string());
         }
 
         public string get_oauth_timestamp() {
@@ -178,12 +174,11 @@ namespace Publishing.RESTSupport.OAuth1 {
             }
         }
 
-
-        public override void execute() throws Spit.Publishing.PublishingError {
+        public override async void execute_async() throws Spit.Publishing.PublishingError {
             var signature = ((Session) get_parent_session()).sign_transaction(this);
             add_argument("oauth_signature", signature);
 
-            base.execute();
+            yield base.execute_async();
         }
     }
 
@@ -212,7 +207,7 @@ namespace Publishing.RESTSupport.OAuth1 {
         }
 
         public string get_authorization_header_string() {
-            return "OAuth " + Argument.serialize_list(auth_header_fields, true, ", ");
+            return "OAuth " + Argument.serialize_list(auth_header_fields, false, true, ", ");
         }
 
         public void authorize() {
diff --git a/plugins/common/RESTSupport.vala b/plugins/common/RESTSupport.vala
index 0d0a3fb..cc810fe 100644
--- a/plugins/common/RESTSupport.vala
+++ b/plugins/common/RESTSupport.vala
@@ -26,6 +26,9 @@ public abstract class Session {
     private string? endpoint_url = null;
     private Soup.Session soup_session = null;
     private bool transactions_stopped = false;
+    private Bytes? body = null;
+    private Error? transport_error= null;
+    private bool insecure = false;
     
     public signal void wire_message_unqueued(Soup.Message message);
     public signal void authenticated();
@@ -34,7 +37,18 @@ public abstract class Session {
     protected Session(string? endpoint_url = null) {
         this.endpoint_url = endpoint_url;
         soup_session = new Soup.Session ();
-        this.soup_session.ssl_use_system_ca_file = true;
+        if (Environment.get_variable("SHOTWELL_SOUP_LOG") != null) {
+            var logger = new Soup.Logger(Soup.LoggerLogLevel.BODY);
+            logger.set_request_filter((logger, msg) => {
+                var content_type = msg.get_request_headers().get_content_type(null);
+                if (content_type != null && content_type == "application/octet-stream") {
+                    return Soup.LoggerLogLevel.HEADERS;
+                }
+
+                return Soup.LoggerLogLevel.BODY;
+            });
+            soup_session.add_feature (logger);
+        }
     }
     
     protected void notify_wire_message_unqueued(Soup.Message message) {
@@ -64,19 +78,33 @@ public abstract class Session {
         return transactions_stopped;
     }
     
-    public void send_wire_message(Soup.Message message) {
-        if (are_transactions_stopped())
+    public async void send_wire_message_async(Soup.Message message) {
+        if (are_transactions_stopped()) {
             return;
+        }
 
-        soup_session.request_unqueued.connect(notify_wire_message_unqueued);
-        soup_session.send_message(message);
-        
-        soup_session.request_unqueued.disconnect(notify_wire_message_unqueued);
+        try {
+            this.body = yield soup_session.send_and_read_async(message, GLib.Priority.DEFAULT, null);
+        } catch (Error error) {
+            debug ("Failed to send_and_read: %s", error.message);
+            this.transport_error = error;
+        }
     }
 
     public void set_insecure () {
-        this.soup_session.ssl_use_system_ca_file = false;
-        this.soup_session.ssl_strict = false;
+        this.insecure = true;
+    }
+
+    public bool get_is_insecure() {
+        return this.insecure;
+    }
+
+    public Error? get_transport_error() {
+        return this.transport_error;
+    }
+
+    public Bytes? get_body() {
+        return this.body;
     }
 }
 
@@ -123,11 +151,19 @@ public class Argument {
         this.value = value;
     }
 
-    public static string serialize_list(Argument[] args, bool escape = false, string? separator = "&") {
+    public static string serialize_for_sbs(Argument[] args) {
+        return Argument.serialize_list(args, true, false, "&");
+    }
+
+    public static string serialize_for_authorization_header(Argument[] args) {
+        return Argument.serialize_list(args, false, true, ", ");
+    }
+
+    public static string serialize_list(Argument[] args, bool encode, bool escape, string? separator) {
         var builder = new StringBuilder("");
 
         foreach (var arg in args) {
-            builder.append(arg.to_string(escape));
+            builder.append(arg.to_string(escape, encode));
             builder.append(separator);
         }
 
@@ -150,8 +186,10 @@ public class Argument {
         return sorted_args.to_array();
     }
 
-    public string to_string (bool escape = false) {
-        return "%s=%s%s%s".printf (this.key, escape ? "\"" : "", this.value, escape ? "\"" : "");
+    public string to_string (bool escape = false, bool encode = false) {
+        return "%s=%s%s%s".printf (this.key, escape ? "\"" : "",
+            encode ? GLib.Uri.escape_string(this.value) : this.value,
+            escape ? "\"" : "");
     }
 }
 
@@ -160,13 +198,12 @@ public class Transaction {
     private bool is_executed = false;
     private weak Session parent_session = null;
     private Soup.Message message = null;
-    private int bytes_written = 0;
-    private Spit.Publishing.PublishingError? err = null;
+    private uint bytes_written = 0;
+    private ulong request_length;
     private string? endpoint_url = null;
     private bool use_custom_payload;
     
-    public signal void chunk_transmitted(int bytes_written_so_far, int total_bytes);
-    public signal void network_error(Spit.Publishing.PublishingError err);
+    public signal void chunk_transmitted(uint bytes_written_so_far, uint total_bytes);
     public signal void completed();
 
     
@@ -188,31 +225,16 @@ public class Transaction {
         message = new Soup.Message(method.to_string(), endpoint_url);
     }
 
-    private void on_wrote_body_data(Soup.Buffer written_data) {
-        bytes_written += (int) written_data.length;
-        while (Gtk.events_pending()) {
-            Gtk.main_iteration();
-        }
-        chunk_transmitted(bytes_written, (int) message.request_body.length);
-    }
-
-    private void on_message_unqueued(Soup.Message message) {
-        if (this.message != message)
-            return;
-
-        try {
-            check_response(message);
-        } catch (Spit.Publishing.PublishingError err) {
-            warning("Publishing error: %s", err.message);
-            warning("response validation failed. bad response = '%s'.", get_response());
-            this.err = err;
-        }
+    private void on_wrote_body_data(Soup.Message message, uint chunk_size) {
+        bytes_written += chunk_size;
+        chunk_transmitted(bytes_written, (uint)request_length);
     }
 
     /* Texts copied from epiphany */
     public string detailed_error_from_tls_flags (out TlsCertificate cert) {
         TlsCertificateFlags tls_errors;
-        this.message.get_https_status (out cert, out tls_errors);
+        cert = this.message.get_tls_peer_certificate();
+        tls_errors = this.message.get_tls_peer_certificate_errors();
 
         var list = new Gee.ArrayList<string> ();
         if (TlsCertificateFlags.BAD_IDENTITY in tls_errors) {
@@ -263,38 +285,38 @@ public class Transaction {
   }
 
     protected void check_response(Soup.Message message) throws Spit.Publishing.PublishingError {
-        switch (message.status_code) {
-            case Soup.KnownStatusCode.OK:
-            case Soup.KnownStatusCode.CREATED: // HTTP code 201 (CREATED) signals that a new
-                                               // resource was created in response to a PUT or POST
-            break;
-            
-            case Soup.KnownStatusCode.CANT_RESOLVE:
-            case Soup.KnownStatusCode.CANT_RESOLVE_PROXY:
+        var transport_error = parent_session.get_transport_error();
+        if (transport_error != null) {
+            if (transport_error is GLib.ResolverError) {
                 throw new Spit.Publishing.PublishingError.NO_ANSWER("Unable to resolve %s (error code %u)",
-                    get_endpoint_url(), message.status_code);
-            
-            case Soup.KnownStatusCode.CANT_CONNECT:
-            case Soup.KnownStatusCode.CANT_CONNECT_PROXY:
+                get_endpoint_url(), message.status_code);
+            }
+            if (transport_error is GLib.IOError) {
                 throw new Spit.Publishing.PublishingError.NO_ANSWER("Unable to connect to %s (error code %u)",
                     get_endpoint_url(), message.status_code);
-            case Soup.KnownStatusCode.SSL_FAILED:
+            }
+            if (transport_error is GLib.TlsError) {
                 throw new Spit.Publishing.PublishingError.SSL_FAILED ("Unable to connect to %s: Secure connection failed",
                     get_endpoint_url ());
+            }
+
+            throw new Spit.Publishing.PublishingError.NO_ANSWER("Failure communicating with %s (error code %u)",
+                get_endpoint_url(), message.status_code);
+        }
+        switch (message.status_code) {
+            case Soup.Status.OK:
+            case Soup.Status.CREATED: // HTTP code 201 (CREATED) signals that a new
+                                               // resource was created in response to a PUT or POST
+            break;
             
             default:
-                // status codes below 100 are used by Soup, 100 and above are defined HTTP codes
-                if (message.status_code >= 100) {
-                    throw new Spit.Publishing.PublishingError.NO_ANSWER("Service %s returned HTTP status code %u %s",
-                        get_endpoint_url(), message.status_code, message.reason_phrase);
-                } else {
-                    throw new Spit.Publishing.PublishingError.NO_ANSWER("Failure communicating with %s (error code %u)",
-                        get_endpoint_url(), message.status_code);
-                }
+                throw new Spit.Publishing.PublishingError.NO_ANSWER("Service %s returned HTTP status code %u %s",
+                    get_endpoint_url(), message.status_code, message.reason_phrase);            
         }
         
         // All valid communication involves body data in the response
-        if (message.response_body.data == null || message.response_body.data.length == 0)
+        var body = parent_session.get_body();
+        if (body == null || body.get_size() == 0)
             throw new Spit.Publishing.PublishingError.MALFORMED_RESPONSE("No response data from %s",
                 get_endpoint_url());
     }
@@ -311,22 +333,26 @@ public class Transaction {
         this.is_executed = is_executed;
     }
 
-    protected void send() throws Spit.Publishing.PublishingError {
-        parent_session.wire_message_unqueued.connect(on_message_unqueued);
-        message.wrote_body_data.connect(on_wrote_body_data);
-        parent_session.send_wire_message(message);
-        
-        parent_session.wire_message_unqueued.disconnect(on_message_unqueued);
-        message.wrote_body_data.disconnect(on_wrote_body_data);
-        
-        if (err != null)
-            network_error(err);
-        else
-            completed();
-        
-        if (err != null)
-            throw err;
-     }
+    private bool on_accecpt_certificate(Soup.Message message, TlsCertificate cert, TlsCertificateFlags errors) {
+        debug ("HTTPS connect error. Will ignore? %s", this.parent_session.get_is_insecure().to_string());
+        return this.parent_session.get_is_insecure();
+    }
+
+    protected async void send_async() throws Spit.Publishing.PublishingError {
+        var id = message.wrote_body_data.connect((message, chunk_size) => {
+            bytes_written = chunk_size;
+
+            chunk_transmitted(bytes_written, (uint)request_length);
+        });
+        message.accept_certificate.connect(on_accecpt_certificate);
+
+        yield parent_session.send_wire_message_async(message);
+        check_response(message);
+
+        message.disconnect(id);
+        message.accept_certificate.disconnect(on_accecpt_certificate);
+        completed();
+    }
 
     public HttpMethod get_method() {
         return HttpMethod.from_string(message.method);
@@ -354,7 +380,8 @@ public class Transaction {
         }
         
         ulong length = (payload_length > 0) ? payload_length : custom_payload.length;
-        message.set_request(payload_content_type, Soup.MemoryUse.COPY, custom_payload.data[0:length]);
+        message.set_request_body_from_bytes(payload_content_type, new Bytes (custom_payload.data[0:length]));
+        this.request_length = length;
 
         use_custom_payload = true;
     }
@@ -364,8 +391,9 @@ public class Transaction {
     // alone and let the Transaction class manage it for you. You should only need
     // to install a new message if your subclass has radically different behavior from
     // normal Transactions -- like multipart encoding.
-    protected void set_message(Soup.Message message) {
+    protected void set_message(Soup.Message message, ulong request_length) {
         this.message = message;
+        this.request_length = request_length;
     }
     
     public bool get_is_executed() {
@@ -377,58 +405,67 @@ public class Transaction {
         return message.status_code;
     }
 
-    public virtual void execute() throws Spit.Publishing.PublishingError {
-        // if a custom payload is being used, we don't need to peform the tasks that are necessary
-        // to prepare a traditional key-value pair REST request; Instead (since we don't
-        // know anything about the custom payload), we just put it on the wire and return
-        if (use_custom_payload) {
-            is_executed = true;
-            send();
-
-            return;
-         }
-         
+    private GLib.Uri? prepare_rest_message() {
         //  REST POST requests must transmit at least one argument
         if (get_method() == HttpMethod.POST)
             assert(arguments.length > 0);
 
         // concatenate the REST arguments array into an HTTP formdata string
-        string formdata_string = "";
+        var formdata_string = new StringBuilder("");
         for (int i = 0; i < arguments.length; i++) {
-            formdata_string += arguments[i].to_string ();
+            formdata_string.append(arguments[i].to_string());
             if (i < arguments.length - 1)
-                formdata_string += "&";
+                formdata_string.append("&");
         }
         
         // for GET requests with arguments, append the formdata string to the endpoint url after a
         // query divider ('?') -- but make sure to save the old (caller-specified) endpoint URL
         // and restore it after the GET so that the underlying Soup message remains consistent
-        string old_url = null;
+        GLib.Uri? old_url = null;
         string url_with_query = null;
         if (get_method() == HttpMethod.GET && arguments.length > 0) {
-            old_url = message.get_uri().to_string(false);
-            url_with_query = get_endpoint_url() + "?" + formdata_string;
-            message.set_uri(new Soup.URI(url_with_query));
+            old_url = message.get_uri();
+            url_with_query = get_endpoint_url() + "?" + formdata_string.str;
+            try {
+                message.set_uri(GLib.Uri.parse(url_with_query, GLib.UriFlags.ENCODED));
+            } catch (Error err) {
+                error ("Invalid uri for service: %s", err.message);
+            }
         } else {
-            message.set_request("application/x-www-form-urlencoded", Soup.MemoryUse.COPY,
-                formdata_string.data);
+            message.set_request_body_from_bytes("application/x-www-form-urlencoded", StringBuilder.free_to_bytes((owned)formdata_string));
         }
 
         is_executed = true;
 
+        return old_url;
+    }
+
+    public virtual async void execute_async() throws Spit.Publishing.PublishingError {
+        // if a custom payload is being used, we don't need to peform the tasks that are necessary
+        // to prepare a traditional key-value pair REST request; Instead (since we don't
+        // know anything about the custom payload), we just put it on the wire and return
+        if (use_custom_payload) {
+            is_executed = true;
+            yield send_async();
+
+            return;
+        }
+
+        var old_url = prepare_rest_message();
+         
         try {
-            debug("sending message to URI = '%s'", message.get_uri().to_string(false));
-            send();
+            debug("sending message to URI = '%s'", message.get_uri().to_string());
+            yield send_async();
         } finally {
             // if old_url is non-null, then restore it
             if (old_url != null)
-                message.set_uri(new Soup.URI(old_url));
+                message.set_uri(old_url);
         }
     }
 
     public string get_response() {
         assert(get_is_executed());
-        return (string) message.response_body.data;
+        return parent_session.get_body() == null ? "" : (string) parent_session.get_body().get_data();
     }
     
     public unowned Soup.MessageHeaders get_response_headers() {
@@ -510,7 +547,7 @@ public class UploadTransaction : Transaction {
         GLib.HashTable<string, string> result =
             new GLib.HashTable<string, string>(GLib.str_hash, GLib.str_equal);
 
-        result.insert("filename", Soup.URI.encode(publishable.get_serialized_file().get_basename(),
+        result.insert("filename", GLib.Uri.escape_string(publishable.get_serialized_file().get_basename(),
             null));
 
         return result;
@@ -520,7 +557,7 @@ public class UploadTransaction : Transaction {
         binary_disposition_table = new_disp_table;
     }
 
-    public override void execute() throws Spit.Publishing.PublishingError {
+    private void prepare_execution() throws Spit.Publishing.PublishingError {
         Argument[] request_arguments = get_arguments();
         assert(request_arguments.length > 0);
 
@@ -529,40 +566,40 @@ public class UploadTransaction : Transaction {
         foreach (Argument arg in request_arguments)
             message_parts.append_form_string(arg.key, arg.value);
 
-        string payload;
-        size_t payload_length;
+        MappedFile? mapped_file = null;
         try {
-            FileUtils.get_contents(publishable.get_serialized_file().get_path(), out payload,
-                out payload_length);
-        } catch (FileError e) {
+            mapped_file = new MappedFile(publishable.get_serialized_file().get_path(), false);
+        } catch (Error e) {
             throw new Spit.Publishing.PublishingError.LOCAL_FILE_ERROR(
                 _("A temporary file needed for publishing is unavailable"));
         }
 
-        int payload_part_num = message_parts.get_length();
-
-        Soup.Buffer bindable_data = new Soup.Buffer(Soup.MemoryUse.COPY, payload.data[0:payload_length]);
         message_parts.append_form_file("", publishable.get_serialized_file().get_path(), mime_type,
-            bindable_data);
+            mapped_file.get_bytes());
 
         unowned Soup.MessageHeaders image_part_header;
-        unowned Soup.Buffer image_part_body;
+        unowned Bytes image_part_body;
+        int payload_part_num = message_parts.get_length() - 1;
         message_parts.get_part(payload_part_num, out image_part_header, out image_part_body);
+        debug ("Image part header %p", image_part_header);
         image_part_header.set_content_disposition("form-data", binary_disposition_table);
 
-        Soup.Message outbound_message =
-            Soup.Form.request_new_from_multipart(get_endpoint_url(), message_parts);
-        // TODO: there must be a better way to iterate over a map
+        var outbound_message = new Soup.Message.from_multipart(get_endpoint_url(), message_parts);
+
         Gee.MapIterator<string, string> i = message_headers.map_iterator();
         bool cont = i.next();
         while(cont) {
             outbound_message.request_headers.append(i.get_key(), i.get_value());
             cont = i.next();
         }
-        set_message(outbound_message);
+        set_message(outbound_message, mapped_file.get_length());
         
         set_is_executed(true);
-        send();
+    }
+
+    public override async void execute_async() throws Spit.Publishing.PublishingError {
+        prepare_execution();
+        yield send_async();
     }
 }
 
@@ -690,9 +727,8 @@ public abstract class BatchUploader {
         this.session = session;
     }
 
-    private void send_files() {
+    private async void send_files_async() throws Spit.Publishing.PublishingError {
         current_file = 0;
-        bool stop = false;
         foreach (Spit.Publishing.Publishable publishable in publishables) {
             GLib.File? file = publishable.get_serialized_file();
             
@@ -710,26 +746,15 @@ public abstract class BatchUploader {
            
             txn.chunk_transmitted.connect(on_chunk_transmitted);
             
-            try {
-                txn.execute();
-            } catch (Spit.Publishing.PublishingError err) {
-                upload_error(err);
-                stop = true;
-            }
+            yield txn.execute_async();
                 
             txn.chunk_transmitted.disconnect(on_chunk_transmitted);           
-            
-            if (stop)
-                break;
-            
+                        
             current_file++;
         }
-        
-        if (!stop)
-            upload_complete(current_file);
     }
-    
-    private void on_chunk_transmitted(int bytes_written_so_far, int total_bytes) {
+
+    private void on_chunk_transmitted(uint bytes_written_so_far, uint total_bytes) {
         double file_span = 1.0 / publishables.length;
         double this_file_fraction_complete = ((double) bytes_written_so_far) / total_bytes;
         double fraction_complete = (current_file * file_span) + (this_file_fraction_complete *
@@ -748,12 +773,14 @@ public abstract class BatchUploader {
     }
     
     protected abstract Transaction create_transaction(Spit.Publishing.Publishable publishable);
-    
-    public void upload(Spit.Publishing.ProgressCallback? status_updated = null) {
+
+    public async int upload_async(Spit.Publishing.ProgressCallback? status_updated = null)  throws Spit.Publishing.PublishingError {
         this.status_updated = status_updated;
 
         if (publishables.length > 0)
-           send_files();
+           yield send_files_async();
+
+        return current_file;
     }
 }
 
diff --git a/plugins/common/WebAuthenticationPane.vala b/plugins/common/WebAuthenticationPane.vala
index 02cffb2..b9f7280 100644
--- a/plugins/common/WebAuthenticationPane.vala
+++ b/plugins/common/WebAuthenticationPane.vala
@@ -19,13 +19,8 @@ namespace Shotwell.Plugins.Common {
         private Gtk.Entry entry;
 
         public void clear() {
-            try {
-                debug("Clearing the data of WebKit...");
-                this.webview.get_website_data_manager().clear.begin(WebKit.WebsiteDataTypes.ALL, (GLib.TimeSpan)0);
-            } catch (Error e) {
-                // Do nothing
-                message("Failed to clear data: %s", e.message);
-            }
+            debug("Clearing the data of WebKit...");
+            this.webview.get_website_data_manager().clear.begin(WebKit.WebsiteDataTypes.ALL, (GLib.TimeSpan)0);
         }
 
         public override void constructed () {
@@ -44,7 +39,6 @@ namespace Shotwell.Plugins.Common {
             box.pack_start (entry, false, false, 6);
 
             this.webview = new WebKit.WebView ();
-            this.webview.get_settings ().enable_plugins = false;
 
             this.webview.load_changed.connect (this.on_page_load_changed);
             this.webview.load_failed.connect (this.on_page_load_failed);