Skip to content
GitLab
Explore
Sign in
Register
Primary navigation
Search or go to…
Project
soledad
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Container registry
Model registry
Operate
Environments
Monitor
Incidents
Service Desk
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
This is an archived project. Repository and other project resources are read-only.
Show more breadcrumbs
leap
soledad
Commits
1bce0175
Verified
Commit
1bce0175
authored
7 years ago
by
Victor
Browse files
Options
Downloads
Patches
Plain Diff
[feature] add a producer protocol for upstream
-- Resolves:
#8809
parent
56a4ffd9
No related branches found
No related tags found
1 merge request
!172
Upstreaming blobs
Changes
2
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
src/leap/soledad/client/_db/blobs/__init__.py
+6
-13
6 additions, 13 deletions
src/leap/soledad/client/_db/blobs/__init__.py
src/leap/soledad/client/_db/blobs/upstream_producer.py
+89
-0
89 additions, 0 deletions
src/leap/soledad/client/_db/blobs/upstream_producer.py
with
95 additions
and
13 deletions
src/leap/soledad/client/_db/blobs/__init__.py
+
6
−
13
View file @
1bce0175
...
@@ -50,6 +50,7 @@ from .sql import SyncStatus
...
@@ -50,6 +50,7 @@ from .sql import SyncStatus
from
.sql
import
Priority
from
.sql
import
Priority
from
.sql
import
SQLiteBlobBackend
from
.sql
import
SQLiteBlobBackend
from
.sync
import
BlobsSynchronizer
from
.sync
import
BlobsSynchronizer
from
.upstream_producer
import
BlobsUpstreamProducer
from
.errors
import
(
from
.errors
import
(
BlobAlreadyExistsError
,
MaximumRetriesError
,
BlobAlreadyExistsError
,
MaximumRetriesError
,
RetriableTransferError
,
BlobNotFoundError
,
InvalidFlagsError
)
RetriableTransferError
,
BlobNotFoundError
,
InvalidFlagsError
)
...
@@ -498,25 +499,17 @@ class BlobManager(BlobsSynchronizer):
...
@@ -498,25 +499,17 @@ class BlobManager(BlobsSynchronizer):
@defer.inlineCallbacks
@defer.inlineCallbacks
def
_upstream
(
self
,
blobs_id_list
,
namespace
=
''
):
def
_upstream
(
self
,
blobs_id_list
,
namespace
=
''
):
local
,
secret
=
self
.
local
,
self
.
secret
uri
=
urljoin
(
self
.
remote_stream
,
self
.
user
)
uri
=
urljoin
(
self
.
remote_stream
,
self
.
user
)
params
=
{
'
namespace
'
:
namespace
}
if
namespace
else
None
params
=
{
'
namespace
'
:
namespace
}
if
namespace
else
None
sizes
=
yield
self
.
local
.
get_size_list
(
blobs_id_list
,
namespace
)
sizes
=
yield
self
.
local
.
get_size_list
(
blobs_id_list
,
namespace
)
convert
=
get_unarmored_ciphertext_size
convert
=
get_unarmored_ciphertext_size
sizes
=
map
(
lambda
(
x
,
y
):
(
x
,
convert
(
y
)),
sizes
)
sizes
=
map
(
lambda
(
blob_id
,
size
):
(
blob_id
,
convert
(
size
)),
sizes
)
data
=
BytesIO
()
# TODO: stream from db
producer
=
BlobsUpstreamProducer
(
local
,
sizes
,
namespace
,
secret
)
data
.
write
(
json
.
dumps
(
sizes
)
+
'
\n
'
)
for
blob_id
in
blobs_id_list
:
blob_fd
=
yield
self
.
local
.
get
(
blob_id
,
namespace
=
namespace
)
doc_info
=
DocInfo
(
blob_id
,
FIXED_REV
)
crypter
=
BlobEncryptor
(
doc_info
,
blob_fd
,
secret
=
self
.
secret
,
armor
=
False
)
fd
=
yield
crypter
.
encrypt
()
data
.
write
(
fd
.
read
())
data
.
seek
(
0
)
params
=
{
'
namespace
'
:
namespace
}
if
namespace
else
{}
params
=
{
'
namespace
'
:
namespace
}
if
namespace
else
{}
params
[
'
direction
'
]
=
'
upload
'
params
[
'
direction
'
]
=
'
upload
'
response
=
yield
self
.
_client
.
post
(
uri
,
data
=
data
,
params
=
params
)
response
=
yield
self
.
_client
.
post
(
uri
,
data
=
producer
,
params
=
params
)
check_http_status
(
response
.
code
,
blob_id
)
check_http_status
(
response
.
code
,
'
stream
'
)
logger
.
info
(
"
Finished stream up: %s
"
%
(
blobs_id_list
,))
logger
.
info
(
"
Finished stream up: %s
"
%
(
blobs_id_list
,))
@defer.inlineCallbacks
@defer.inlineCallbacks
...
...
This diff is collapsed.
Click to expand it.
src/leap/soledad/client/_db/blobs/upstream_producer.py
0 → 100644
+
89
−
0
View file @
1bce0175
# -*- coding: utf-8 -*-
# upstream_producer.py
# Copyright (C) 2017 LEAP
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import
json
from
zope.interface
import
implementer
from
twisted.internet
import
defer
from
twisted.internet
import
reactor
from
twisted.web.iweb
import
IBodyProducer
from
twisted.web.iweb
import
UNKNOWN_LENGTH
from
leap.soledad.client._crypto
import
DocInfo
from
leap.soledad.client._crypto
import
BlobEncryptor
FIXED_REV
=
'
ImmutableRevision
'
# Blob content is immutable
@implementer
(
IBodyProducer
)
class
BlobsUpstreamProducer
(
object
):
"""
Blob producer for upload streams.
"""
def
__init__
(
self
,
database
,
blobs_lengths
,
namespace
,
secret
):
"""
Initialize the upload streamer.
:param database: Local blobs SQLCipher backend instance
:type database: .sql.SQLiteBlobBackend
:param blobs_lengths: List of blobs with ids and sizes
:type blobs_lengths: [(blob_id:str, size:int)]
:param namespace: Namespace which this stream belongs
:type namespace: str
:param secret: The secret used to encrypt blobs.
:type secret: str
"""
self
.
blobs_lengths
=
blobs_lengths
self
.
db
=
database
self
.
length
=
UNKNOWN_LENGTH
self
.
pause
=
False
self
.
stop
=
False
self
.
namespace
=
namespace
self
.
secret
=
secret
@defer.inlineCallbacks
def
startProducing
(
self
,
consumer
):
"""
Write blobs to the consumer.
:param consumer: Any IConsumer provider.
:type consumer: twisted.internet.interfaces.IConsumer
:return: A Deferred that fires when production ends.
:rtype: twisted.internet.defer.Deferred
"""
consumer
.
write
(
json
.
dumps
(
self
.
blobs_lengths
)
+
'
\n
'
)
for
blob_id
,
_
in
self
.
blobs_lengths
:
blob_fd
=
yield
self
.
db
.
get
(
blob_id
,
namespace
=
self
.
namespace
)
doc_info
=
DocInfo
(
blob_id
,
FIXED_REV
)
crypter
=
BlobEncryptor
(
doc_info
,
blob_fd
,
secret
=
self
.
secret
,
armor
=
False
)
fd
=
yield
crypter
.
encrypt
()
consumer
.
write
(
fd
.
read
())
def
sleep
(
self
,
secs
):
d
=
defer
.
Deferred
()
reactor
.
callLater
(
secs
,
d
.
callback
,
None
)
return
d
def
pauseProducing
(
self
):
self
.
pause
=
True
def
stopProducing
(
self
):
self
.
stop
=
True
def
resumeProducing
(
self
):
self
.
pause
=
False
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment