[WIP] Implementation of Python Stats API#149
[WIP] Implementation of Python Stats API#149liyanhui1228 merged 55 commits intocensus-instrumentation:masterfrom
Conversation
|
Thanks for your pull request. It looks like this may be your first contribution to a Google open source project (if not, look below for help). Before we can look at your pull request, you'll need to sign a Contributor License Agreement (CLA). 📝 Please visit https://cla.developers.google.com/ to sign. Once you've signed (or fixed any issues), please reply here (e.g. What to do if you already signed the CLAIndividual signers
Corporate signers
|
|
I signed it! |
|
I signed it! |
|
CLAs look good, thanks! |
examples/stats/basic_prototype.py
Outdated
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. |
There was a problem hiding this comment.
Please add this license to all the other files.
opencensus/stats/aggregation.py
Outdated
| def __init__(self, boundaries=None, distribution=None, aggregation_type="distribution"): | ||
| super().__init__(aggregation_type, boundaries) | ||
| self.aggregation_type = aggregation_type | ||
| '''self.boundaries = list(bucket_boundaries.BucketBoundaries(boundaries) or [])''' |
opencensus/stats/aggregation.py
Outdated
| self.boundaries = bucket_boundaries.BucketBoundaries(boundaries) | ||
| self.distribution = dict(distribution or {}) | ||
|
|
||
| def get_aggregation(self): |
There was a problem hiding this comment.
In Python we don't really use getters and setters like in Java, because we can get the variable just by calling self.aggregation_type in this case. Usually when we use getter method, we add a @property decorator before the method signature, and the variable the getter returned is a private variable in this class which has a _ before the variable name. Then with this decorated getter method, calling self.aggregation_type will return the value of the private variable self._aggregation_type in this class.
For more details look at this link: https://www.python-course.eu/python3_properties.php
| self.resource = client.resource('global', {}) | ||
|
|
||
| def emit(self, views, datapoint=None): | ||
| name = 'projects/{}'.format(self.project_id) |
There was a problem hiding this comment.
Move this line to __init__ so you don't need to construct the name every time you call emit.
|
|
||
| def emit(self, views, datapoint=None): | ||
| name = 'projects/{}'.format(self.project_id) | ||
| '''metric = {}''' |
setup.py
Outdated
|
|
||
| install_requires = [ | ||
| 'google-cloud-trace>=0.17.0, <0.18dev', | ||
| 'google-cloud-monitoring>=0.28.1' |
There was a problem hiding this comment.
google-cloud-monitoring>=0.28.1, <0.29,
| else: | ||
| return "key is not in map" | ||
|
|
||
|
|
There was a problem hiding this comment.
Clean up the extra new lines.
| return False | ||
|
|
||
|
|
||
|
|
There was a problem hiding this comment.
Clean up extra new lines.
| metrics[metric_type] = metric_label | ||
| labels.append(metric_label) | ||
| return metrics | ||
|
|
There was a problem hiding this comment.
Clean up extra new lines.
|
|
||
| def translate_to_stackdriver(self, views): | ||
| metrics = {} | ||
| labels = [] |
There was a problem hiding this comment.
What's the purpose of labels?
| return 'Successfully wrote time series.' | ||
|
|
||
| def set_resource(self, type_, labels): | ||
| return self.client.resource(type_, labels) |
There was a problem hiding this comment.
Set 'global' as the default type_ here.
opencensus/trace/tracers/__init__.py
Outdated
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
| from opencensus.trace import tracer as tracer_module |
There was a problem hiding this comment.
Why changing this file?
|
FYI I updated the |
opencensus/tags/tag_value.py
Outdated
| def value(self): | ||
| return self._value | ||
|
|
||
| def is_value(self, value): |
There was a problem hiding this comment.
Rename to check_value?
opencensus/tags/tag_map.py
Outdated
| if key in self.map: | ||
| return self.map[key] | ||
| else: | ||
| return "key is not in map" |
There was a problem hiding this comment.
If you want to emit a warning message, it should be either in the msg of an exception or a logging msg, but should not be directly returned. Because it will mess up with your intended return type of the tag value. In this case, probably change to something like:
value = self.map.get(key, None)
if value is None:
raise KeyError('Key is not in map.')
return value
opencensus/tags/tag_map.py
Outdated
| if key in self.map: | ||
| return True | ||
| else: | ||
| return False |
There was a problem hiding this comment.
Simplify this with:
return key in self.map
opencensus/tags/tag_map.py
Outdated
| if map is None: | ||
| self.map = {} | ||
| else: | ||
| self.map = map |
There was a problem hiding this comment.
self.map = map if map is not None else {}
opencensus/tags/tag_map.py
Outdated
| self.map = map | ||
| if tags is not None: | ||
| self.tags = tags | ||
| print(self.tags) |
There was a problem hiding this comment.
remove print statement?
opencensus/tags/tag_map.py
Outdated
| self.tags = tags | ||
| print(self.tags) | ||
| for tag in self.tags: | ||
| print(tag) |
opencensus/tags/tag_key.py
Outdated
| def name(self): | ||
| return self._name | ||
|
|
||
| def is_valid(self, name): |
There was a problem hiding this comment.
Rename to check_name?
opencensus/tags/tag_context.py
Outdated
|
|
||
| def get_tag_value(self, key): | ||
| if key in self._tags: | ||
| return self._tags[key] |
There was a problem hiding this comment.
return self._tags.get(key, None)
venv/include/python2.7
Outdated
| @@ -0,0 +1 @@ | |||
| /usr/local/Cellar/python/2.7.14_3/Frameworks/Python.framework/Versions/2.7/include/python2.7 No newline at end of file | |||
There was a problem hiding this comment.
Use this command git rm --cached venv/include/python2.7 to ignore the previously committed file and it will be ignored after that.
There was a problem hiding this comment.
Please remove the venv/ files from the commits.
venv/pip-selfcheck.json
Outdated
| @@ -0,0 +1 @@ | |||
| {"last_check":"2018-02-13T00:28:25Z","pypi_version":"9.0.1"} No newline at end of file | |||
wkiser
left a comment
There was a problem hiding this comment.
Thanks for the PR! I think it might make sense to split this into a few smaller PRs since there is a lot going on. Maybe start with the base classes + related tests?
opencensus/stats/aggregation.py
Outdated
| class BaseAggregation(object): | ||
|
|
||
| def __init__(self, aggregation_type=None, buckets=None): | ||
| if aggregation_type is not None: |
There was a problem hiding this comment.
aggregation_type can probably be a class level variable since it will be the same for all objects of the type.
opencensus/stats/aggregation.py
Outdated
| else: | ||
| self._aggregation_type = "none" | ||
|
|
||
| if buckets is not None: |
There was a problem hiding this comment.
I think self._buckets = buckets or [] is more readable, but I'm not sure if that is against this repo's style guide.
opencensus/stats/aggregation.py
Outdated
| def __init__(self, count=None, aggregation_type="count"): | ||
| super().__init__(aggregation_type) | ||
| self._aggregation_type = aggregation_type | ||
| if count is not None: |
There was a problem hiding this comment.
you can make 0 the default value for count in the initializer: count=0,...
| self._count_data = count_data | ||
|
|
||
| def add_sample(self, value): | ||
| self._count_data = self._count_data + 1 |
opencensus/stats/aggregation_data.py
Outdated
| self._sum_data = sum_data | ||
|
|
||
| def add_sample(self, value): | ||
| self._sum_data = self._sum_data + value |
| self.client = client | ||
| self.project_id = client.project | ||
| self.resource = client.resource('global', {}) | ||
| name = 'projects/{}'.format(self.project_id) |
| self.client.resource = self.set_resource(type_='global', labels= {'project_id': self.project_id}) | ||
| self.client.write_point(metric, self.client.resource, datapoint, datetime.utcnow() + timedelta(seconds=60), datetime.utcnow()) | ||
| self.client.time_series(metric, self.client.resource, datapoint, datetime.utcnow() + timedelta(seconds=60), datetime.utcnow()) | ||
| return 'Successfully wrote time series.' |
There was a problem hiding this comment.
I would log this rather than return it.
opencensus/stats/aggregation_data.py
Outdated
| return self._count_data | ||
|
|
||
| class DistributionAggregationData(BaseAggregationData): | ||
| def __init__(self, mean_data, count_data, min, max, sum_of_sqd_deviations, counts_per_bucket, bounds): |
There was a problem hiding this comment.
might want to do min_, max_ since min and max are reserved words
| self.client.time_series(metric, self.client.resource, datapoint, datetime.utcnow() + timedelta(seconds=60), datetime.utcnow()) | ||
| return 'Successfully wrote time series.' | ||
|
|
||
| def set_resource(self, type_='global', labels={}): |
There was a problem hiding this comment.
weird python quirk, but it's considered dangerous to use an empty dict as the default value to a function. I would replace with labels=None and create a new empty dict inside the function body is labels is None.
opencensus/tags/tag_map.py
Outdated
| class TagMap(object): | ||
|
|
||
| def __init__(self, tags=None, map=None): | ||
| self.map = map if map is not None else {} |
There was a problem hiding this comment.
can do self.map = map or {}
|
Adding @songy23 for sanity check. |
| self._registered_views[measure.name] = view | ||
| if registered_measure is None: | ||
| self._registered_measures[measure.name] = measure | ||
| self._map[view.measure.name] = ViewData(view=view, start_time=timestamp, end_time=timestamp) |
There was a problem hiding this comment.
Adding comment based on our discussion here for future reference: self._map should be implemented as a multi-map instead of a single map, as there could be multiple view data using the same measure. And in this way there will only be one view data because everything before are overwrote.
songy23
left a comment
There was a problem hiding this comment.
First pass - I haven't looked at the tag part.
opencensus/stats/aggregation.py
Outdated
| return self._count | ||
|
|
||
|
|
||
| class MeanAggregation(BaseAggregation): |
There was a problem hiding this comment.
Mean Aggregation should be removed now: census-instrumentation/opencensus-specs#56.
opencensus/stats/aggregation_data.py
Outdated
| return self._count_data | ||
|
|
||
|
|
||
| class MeanAggregationData(BaseAggregationData): |
| class BucketBoundaries(object): | ||
|
|
||
| def __init__(self, boundaries=None): | ||
| self._boundaries = list(boundaries or []) |
There was a problem hiding this comment.
You should also check the boundaries are in ascending order.
| class MeasureToViewMap(object): | ||
|
|
||
| def __init__(self): | ||
| self._map = {} |
opencensus/stats/view_data.py
Outdated
| self._view = view | ||
| self._start_time = start_time | ||
| self._end_time = end_time | ||
| self._rows = rows if rows is not None else {} |
There was a problem hiding this comment.
FWIW rows and tag_value_aggregation_map are the same thing.
liyanhui1228
left a comment
There was a problem hiding this comment.
Mostly LGTM excepts the Circle CI needs to be green before we merge. Please fix the lint and coverage tests. And don't forget to remove the unnecessary files like venv/. Use git rm --cached venv/include/python2.7 to remove the committed files from the commits.
songy23
left a comment
There was a problem hiding this comment.
Please consider moving the exporter code to another PR.
| :type count_data: int | ||
| :param count_data: the count value of the distribution | ||
|
|
||
| :type min_: double |
There was a problem hiding this comment.
I believe min is a keyword in python which is why it is not used, would you recommend to use the keyword instead?
There was a problem hiding this comment.
Got it, I'm fine with leaving it as-is. I was just wondering why these two were different from other attributes.
| :type min_: double | ||
| :param min_: the minimum value of the distribution | ||
|
|
||
| :type max_: double |
| :rtype: bool | ||
| :returns: True if it valid, else returns False | ||
| """ | ||
| if (len(name) > 0) and (len(name) <= 255): |
| """The current value""" | ||
| return self._value | ||
|
|
||
| def is_valid_value(self, value): |
There was a problem hiding this comment.
Same here, you should check if it's an ASCII string.
| return self._rows | ||
|
|
||
| @property | ||
| def tag_value_aggregation_map(self): |
There was a problem hiding this comment.
The key of this map should be different combinations of TagValues, rather than single TagValue.
opencensus/stats/view_data.py
Outdated
| def record(self, context, value, timestamp): | ||
| """records the view data against context""" | ||
| tag_values = self.get_tag_values(tags=self.get_tag_map(context), columns=self.view.columns) | ||
| for tag_value in tag_values: |
There was a problem hiding this comment.
Same here, you should check and update entry for unique combination of TagValues instead of going through each one.
|
|
||
| def get_view(self, view_name, timestamp): | ||
| """get the View Data from the given View name""" | ||
| view = self.get_view_data(view_name) |
There was a problem hiding this comment.
This view is actually a ViewData?
| if view is None: | ||
| return None | ||
|
|
||
| view_data = ViewData(view=view, start_time=timestamp, end_time=timestamp) |
There was a problem hiding this comment.
IIUC the variable view is a ViewData rather than a View. How do you set the rows/aggregation map for the ViewData?
opencensus-traces.json
Outdated
| @@ -0,0 +1 @@ | |||
| {"traceId": "f8739df974a4481f98748cd92b27177d", "spans": [{"displayName": {"value": "child_span", "truncated_byte_count": 0}, "spanId": "739fba56569f4106", "startTime": "2018-05-07T21:04:16.261253Z", "endTime": "2018-05-07T21:04:18.266410Z", "childSpanCount": 0, "kind": 0, "parentSpanId": "c3e7efa94eb747d3"}, {"displayName": {"value": "root_span", "truncated_byte_count": 0}, "spanId": "c3e7efa94eb747d3", "startTime": "2018-05-07T21:04:14.256056Z", "endTime": "2018-05-07T21:04:18.266438Z", "childSpanCount": 1, "kind": 0, "parentSpanId": "6e0c63257de34c92"}]} No newline at end of file | |||
There was a problem hiding this comment.
Is this file accidentally committed? Remove it?
| self._count_data = count_data | ||
|
|
||
| def add_sample(self, value): | ||
| self._count_data = self._count_data + 1 |
opencensus/stats/aggregation_data.py
Outdated
| :param count_data: the count value of the distribution | ||
|
|
||
| :type min_: double | ||
| :type min_: min |
opencensus/stats/aggregation_data.py
Outdated
| :param min_: the minimum value of the distribution | ||
|
|
||
| :type max_: double | ||
| :type max_: max |
liyanhui1228
left a comment
There was a problem hiding this comment.
Please fix the final review comments and we'll be good to merge.
examples/stats/basic_prototype.py
Outdated
|
|
||
|
|
||
|
|
||
|
|
There was a problem hiding this comment.
Two blank lines between functions.
opencensus-traces.json
Outdated
| @@ -0,0 +1 @@ | |||
| {"traceId": "f8739df974a4481f98748cd92b27177d", "spans": [{"displayName": {"value": "child_span", "truncated_byte_count": 0}, "spanId": "739fba56569f4106", "startTime": "2018-05-07T21:04:16.261253Z", "endTime": "2018-05-07T21:04:18.266410Z", "childSpanCount": 0, "kind": 0, "parentSpanId": "c3e7efa94eb747d3"}, {"displayName": {"value": "root_span", "truncated_byte_count": 0}, "spanId": "c3e7efa94eb747d3", "startTime": "2018-05-07T21:04:14.256056Z", "endTime": "2018-05-07T21:04:18.266438Z", "childSpanCount": 1, "kind": 0, "parentSpanId": "6e0c63257de34c92"}]} No newline at end of file | |||
venv/.Python
Outdated
| @@ -0,0 +1 @@ | |||
| /usr/local/Cellar/python/2.7.14_3/Frameworks/Python.framework/Versions/2.7/Python No newline at end of file | |||
venv/pip-selfcheck.json
Outdated
| @@ -0,0 +1 @@ | |||
| {"last_check":"2018-02-13T00:28:25Z","pypi_version":"9.0.1"} No newline at end of file | |||
|
We can ignore the CLA, merging... |
No description provided.