github/goreplay
4
0
Fork 0
mirror of https://github.com/buger/goreplay.git synced 2024-04-21 12:32:02 +00:00
Code Issues Releases Wiki Activity

Merge branch 'master' into s3

Browse Source
This commit is contained in:
Leonid Bugaev
2016-12-19 20:25:06 +03:00
parent ddf2dc2876 c4869d9338
commit cd51e09fcd
46 changed files with 2586 additions and 636 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+5
View File
@@ -6,6 +6,7 @@
*.bin
*.gz
*.zip
*.class
@@ -15,3 +16,7 @@
gor
*.mprof
*.pcap
.DS_Store
.gitmodules
+15
View File
@@ -0,0 +1,15 @@
[submodule "vendor/github.com/mattbaird/elastigo"]
path = vendor/github.com/mattbaird/elastigo
url = https://github.com/mattbaird/elastigo
[submodule "vendor/github.com/araddon/gou"]
path = vendor/github.com/araddon/gou
url = https://github.com/araddon/gou
[submodule "vendor/github.com/bitly/go-hostpool"]
path = vendor/github.com/bitly/go-hostpool
url = https://github.com/bitly/go-hostpool
[submodule "vendor/github.com/bmizerany/assert"]
path = vendor/github.com/bmizerany/assert
url = https://github.com/bmizerany/assert
[submodule "vendor/github.com/google/gopacket"]
path = vendor/github.com/google/gopacket
url = https://github.com/google/gopacket
COMM-LICENSE
+22 -24
View File
@@ -2,7 +2,7 @@ END-USER LICENSE AGREEMENT
------------------------------------------------------------------------------
IMPORTANT: THIS SOFTWARE END-USER LICENSE AGREEMENT ("EULA") IS A LEGAL AGREEMENT (“Agreement”) BETWEEN YOU (THE CUSTOMER, EITHER AS AN INDIVIDUAL OR, IF PURCHASED OR OTHERWISE ACQUIRED BY OR FOR AN ENTITY, AS AN ENTITY) AND REPLAY SOFTWARE. READ IT CAREFULLY BEFORE COMPLETING THE INSTALLATION PROCESS AND USING GOR PRO AND RELATED SOFTWARE COMPONENTS (“SOFTWARE”). IT PROVIDES A LICENSE TO USE THE SOFTWARE AND CONTAINS WARRANTY INFORMATION AND LIABILITY DISCLAIMERS. BY INSTALLING AND USING THE SOFTWARE, YOU ARE CONFIRMING YOUR ACCEPTANCE OF THE SOFTWARE AND AGREEING TO BECOME BOUND BY THE TERMS OF THIS AGREEMENT.
IMPORTANT: THIS SOFTWARE END-USER LICENSE AGREEMENT ("EULA") IS A LEGAL AGREEMENT (“Agreement”) BETWEEN YOU (THE CUSTOMER, EITHER AS AN INDIVIDUAL OR, IF PURCHASED OR OTHERWISE ACQUIRED BY OR FOR AN ENTITY, AS AN ENTITY) AND GoReplay LLC. READ IT CAREFULLY BEFORE COMPLETING THE INSTALLATION PROCESS AND USING GOREPLAY PRO AND RELATED SOFTWARE COMPONENTS (“SOFTWARE”). IT PROVIDES A LICENSE TO USE THE SOFTWARE AND CONTAINS WARRANTY INFORMATION AND LIABILITY DISCLAIMERS. BY INSTALLING AND USING THE SOFTWARE, YOU ARE CONFIRMING YOUR ACCEPTANCE OF THE SOFTWARE AND AGREEING TO BECOME BOUND BY THE TERMS OF THIS AGREEMENT.
------------------------------------------------------------------------------
@@ -12,21 +12,19 @@ In order to use the Software under this Agreement, you must receive a “Source
1.1 General Use. This Agreement grants you a non-exclusive, non-transferable, limited license to the use rights for the Software, without the right to grant sublicenses, subject to the terms and conditions in this Agreement. The Software is licensed, not sold.
1.2 Unlimited Organization License. If you purchased an Organization License (included with the Gor Pro Software), you may install the Software on an unlimited number of Hosts. “Host” means any physical or virtual machine which is controlled by you. You may also run an unlimited number of Workers. “Worker” means a thread within a Gor server process which executes jobs. You may concurrently run the software on an unlimited number of Hosts, with each host running an unlimited number of Workers.
1.2 Unlimited Organization License. If you purchased an Organization License (included with the GoReplay Pro Software), you may install the Software on an unlimited number of Hosts. “Host” means any physical or virtual machine which is controlled by you. You may concurrently run the software on an unlimited number of Hosts.
1.3 Limited Organization License. If you purchased an Organization License (included with the Gor Enterprise Software), you may install the Software on an unlimited number of Hosts. “Host” means any physical or virtual machine which is controlled by you. The aggregate number of Workers run by the hosts must not exceed the maximum number of Workers authorized at the time of purchase. “Worker” means a running Gor instance which intercept or replay traffic. In order to run additional Workers, you must purchase an additional allowance from Replay Software.
1.3 Appliance License. If you purchased an Appliance License, you may distribute the Software in any applications, frameworks, or elements (collectively referred to as an “Application” or “Applications”) that you develop using the Software in accordance with this EULA, provided that such distribution does not violate the restrictions set forth in section 3 of this EULA. You must not remove, obscure or interfere with any copyright, acknowledgment, attribution, trademark, warning or disclaimer statement affixed to, incorporated in or otherwise applied in connection with the Software. You are required to ensure that the Software is not reused by or with any applications other than those with which you distribute it as permitted herein. For example, if You install the Software on a customer’s server, that customer is not permitted to use the Software independently of your Application. You must inform GoReplay LLC of your knowledge of any infringing use of the Software by any of your customers. You are liable for compliance by those third parties with the terms and conditions of this EULA. You will not owe GoReplay LLC any royalties for your distribution of the Software in accordance with this EULA.
1.4 Appliance License. If you purchased an Appliance License, you may distribute the Software in any applications, frameworks, or elements (collectively referred to as an “Application” or “Applications”) that you develop using the Software in accordance with this EULA, provided that such distribution does not violate the restrictions set forth in section 3 of this EULA. You must not remove, obscure or interfere with any copyright, acknowledgment, attribution, trademark, warning or disclaimer statement affixed to, incorporated in or otherwise applied in connection with the Software. You are required to ensure that the Software is not reused by or with any applications other than those with which you distribute it as permitted herein. For example, if You install the Software on a customer’s server, that customer is not permitted to use the Software independently of your Application. You must inform Replay Software of your knowledge of any infringing use of the Software by any of your customers. You are liable for compliance by those third parties with the terms and conditions of this EULA. You will not owe Replay Software any royalties for your distribution of the Software in accordance with this EULA.
1.4 Archive Copies. You are entitled to make a reasonable amount of copies of the Software for archival purposes. Each copy must reproduce all copyright and other proprietary rights notices on or in the Software Product.
1.5 Archive Copies. You are entitled to make a reasonable amount of copies of the Software for archival purposes. Each copy must reproduce all copyright and other proprietary rights notices on or in the Software Product.
1.5 Electronic Delivery. All Software and license documentation shall be delivered by electronic means unless otherwise specified on the applicable invoice or at the time of purchase. Software shall be deemed delivered when it is made available for download by you (“Delivery”).
1.6 Electronic Delivery. All Software and license documentation shall be delivered by electronic means unless otherwise specified on the applicable invoice or at the time of purchase. Software shall be deemed delivered when it is made available for download by you (“Delivery”).
2. Modifications. Replay Software shall provide you with source code so that you can create Modifications of the original software. “Modification” means: (a) any addition to or deletion from the contents of a file included in the original Software or previous Modifications created by You, or (b) any new file that contains any part of the original Software or previous Modifications. While you retain all rights to any original work authored by you as part of the Modifications, We continue to own all copyright and other intellectual property rights in the Software.
2. Modifications. GoReplay LLC shall provide you with source code so that you can create Modifications of the original software. “Modification” means: (a) any addition to or deletion from the contents of a file included in the original Software or previous Modifications created by You, or (b) any new file that contains any part of the original Software or previous Modifications. While you retain all rights to any original work authored by you as part of the Modifications, We continue to own all copyright and other intellectual property rights in the Software.
3. Restricted Uses.
3.1 You shall not (and shall not allow any third party to): (a) decompile, disassemble, or otherwise reverse engineer the Software or attempt to reconstruct or discover any source code, underlying ideas, algorithms, file formats or programming interfaces of the Software by any means whatsoever (except and only to the extent that applicable law prohibits or restricts reverse engineering restrictions); (b) distribute, sell, sublicense, rent, lease or use the Software for time sharing, hosting, service provider or like purposes, except as expressly permitted under this Agreement; (c) redistribute the Software or Modifications other than by including the Software or a portion thereof within your own product, which must have substantially different functionality than the Software or Modifications and must not allow any third party to use the Software or Modifications, or any portions thereof, for software development or application development purposes; (d) redistribute the Software as part of a product, "appliance" or "virtual server"; (e) redistribute the Software on any server which is not directly under your control; (f) remove any product identification, proprietary, copyright or other notices contained in the Software; (g) modify any part of the Software, create a derivative work of any part of the Software (except as permitted in Section 4), or incorporate the Software, except to the extent expressly authorized in writing by Replay Software; (h) publicly disseminate performance information or analysis (including, without limitation, benchmarks) from any source relating to the Software; (i) utilize any equipment, device, software, or other means designed to circumvent or remove any form of Source URL or copy protection used by Replay Software in connection with the Software, or use the Software together with any authorization code, Source URL, serial number, or other copy protection device not supplied by Replay Software; (j) use the Software to develop a product which is competitive with any Replay Software product offerings; or (k) use unauthorized Source URLS or keycode(s) or distribute or publish Source URLs or keycode(s), except as may be expressly permitted by Replay Software in writing. If your unique Source URL is ever published, Replay Software reserves the right to terminate your access without notice.
3.1 You shall not (and shall not allow any third party to): (a) decompile, disassemble, or otherwise reverse engineer the Software or attempt to reconstruct or discover any source code, underlying ideas, algorithms, file formats or programming interfaces of the Software by any means whatsoever (except and only to the extent that applicable law prohibits or restricts reverse engineering restrictions); (b) distribute, sell, sublicense, rent, lease or use the Software for time sharing, hosting, service provider or like purposes, except as expressly permitted under this Agreement; (c) redistribute the Software or Modifications other than by including the Software or a portion thereof within your own product, which must have substantially different functionality than the Software or Modifications and must not allow any third party to use the Software or Modifications, or any portions thereof, for software development or application development purposes; (d) redistribute the Software as part of a product, "appliance" or "virtual server"; (e) redistribute the Software on any server which is not directly under your control; (f) remove any product identification, proprietary, copyright or other notices contained in the Software; (g) modify any part of the Software, create a derivative work of any part of the Software (except as permitted in Section 4), or incorporate the Software, except to the extent expressly authorized in writing by GoReplay LLC; (h) publicly disseminate performance information or analysis (including, without limitation, benchmarks) from any source relating to the Software; (i) utilize any equipment, device, software, or other means designed to circumvent or remove any form of Source URL or copy protection used by GoReplay LLC in connection with the Software, or use the Software together with any authorization code, Source URL, serial number, or other copy protection device not supplied by GoReplay LLC; (j) use the Software to develop a product which is competitive with any GoReplay LLC product offerings; or (k) use unauthorized Source URLS or keycode(s) or distribute or publish Source URLs or keycode(s), except as may be expressly permitted by GoReplay LLC in writing. If your unique Source URL is ever published, GoReplay LLC reserves the right to terminate your access without notice.
3.2 UNDER NO CIRCUMSTANCES MAY YOU USE THE SOFTWARE AS PART OF A PRODUCT OR SERVICE THAT PROVIDES SIMILAR FUNCTIONALITY TO THE SOFTWARE ITSELF.
@@ -34,34 +32,34 @@ The Open Source version of the Software (“LGPL Version”) is licensed
under the terms of the GNU Lesser General Public License versions 3.0
(“LGPL”) and not under this EULA.
4. Ownership. Notwithstanding anything to the contrary contained herein, except for the limited license rights expressly provided herein, Replay Software and its suppliers have and will retain all rights, title and interest (including, without limitation, all patent, copyright, trademark, trade secret and other intellectual property rights) in and to the Software and all copies, modifications and derivative works thereof (including any changes which incorporate any of your ideas, feedback or suggestions). You acknowledge that you are obtaining only a limited license right to the Software, and that irrespective of any use of the words “purchase”, “sale” or like terms hereunder no ownership rights are being conveyed to you under this Agreement or otherwise.
4. Ownership. Notwithstanding anything to the contrary contained herein, except for the limited license rights expressly provided herein, GoReplay LLC and its suppliers have and will retain all rights, title and interest (including, without limitation, all patent, copyright, trademark, trade secret and other intellectual property rights) in and to the Software and all copies, modifications and derivative works thereof (including any changes which incorporate any of your ideas, feedback or suggestions). You acknowledge that you are obtaining only a limited license right to the Software, and that irrespective of any use of the words “purchase”, “sale” or like terms hereunder no ownership rights are being conveyed to you under this Agreement or otherwise.
5. Fees and Payment. The Software license fees will be due and payable in full as set forth in the applicable invoice or at the time of purchase. If the Software does not function properly within two weeks of purchase, please contact us within those two weeks for a refund. You shall be responsible for all taxes, withholdings, duties and levies arising from the order (excluding taxes based on the net income of Replay Software).
5. Fees and Payment. The Software license fees will be due and payable in full as set forth in the applicable invoice or at the time of purchase. If the Software does not function properly within two weeks of purchase, please contact us within those two weeks for a refund. You shall be responsible for all taxes, withholdings, duties and levies arising from the order (excluding taxes based on the net income of GoReplay LLC).
6. Support, Maintenance and Services. Subject to the terms and conditions of this Agreement, as set forth in your invoice, and as set forth on the Gor Pro support page (https://github.com/buger/gor/wiki/Pro-Support), support and maintenance services may be included with the purchase of your license subscription.
6. Support, Maintenance and Services. Subject to the terms and conditions of this Agreement, as set forth in your invoice, and as set forth on the GoReplay Pro support page (https://github.com/buger/gor/wiki/Pro-Support), support and maintenance services may be included with the purchase of your license subscription.
7. Term of Agreement.
7.1 Term. This Agreement is effective as of the Delivery of the Software and expires at such time as all license and service subscriptions hereunder have expired in accordance with their own terms (the “Term”). For clarification, the term of your license under this Agreement may be perpetual, limited for Evaluation Version, or designated as a fixed-term license in the Invoice, and shall be specified at your time of purchase. Either party may terminate this Agreement (including all related Invoices) if the other party: (a) fails to cure any material breach of this Agreement within thirty (30) days after written notice of such breach, provided that Replay Software may terminate this Agreement immediately upon any breach of Section 3 or if you exceed any other restrictions contained in Section 1, unless otherwise specified in this agreement; (b) ceases operation without a successor; or (c) seeks protection under any bankruptcy, receivership, trust deed, creditors arrangement, composition or comparable proceeding, or if any such proceeding is instituted against such party (and not dismissed within sixty (60) days)). Termination is not an exclusive remedy and the exercise by either party of any remedy under this Agreement will be without prejudice to any other remedies it may have under this Agreement, by law, or otherwise.
7.1 Term. This Agreement is effective as of the Delivery of the Software and expires at such time as all license and service subscriptions hereunder have expired in accordance with their own terms (the “Term”). For clarification, the term of your license under this Agreement may be perpetual, limited for Evaluation Version, or designated as a fixed-term license in the Invoice, and shall be specified at your time of purchase. Either party may terminate this Agreement (including all related Invoices) if the other party: (a) fails to cure any material breach of this Agreement within thirty (30) days after written notice of such breach, provided that GoReplay LLC may terminate this Agreement immediately upon any breach of Section 3 or if you exceed any other restrictions contained in Section 1, unless otherwise specified in this agreement; (b) ceases operation without a successor; or (c) seeks protection under any bankruptcy, receivership, trust deed, creditors arrangement, composition or comparable proceeding, or if any such proceeding is instituted against such party (and not dismissed within sixty (60) days)). Termination is not an exclusive remedy and the exercise by either party of any remedy under this Agreement will be without prejudice to any other remedies it may have under this Agreement, by law, or otherwise.
7.2 Termination. Upon any termination of this Agreement, you shall cease any and all use of any Software and destroy all copies thereof.
7.3 Expiration of License. Upon the expiration of any term under this Agreement, (a) all Software updates and services pursuant to the license shall cease, (b) you may only continue to run existing installations of the Software, (c) you may not install the Software on any additional Hosts, and (d) any new installation of the Software shall require the purchase of a new license subscription from Replay Software.
7.3 Expiration of License. Upon the expiration of any term under this Agreement, (a) all Software updates and services pursuant to the license shall cease, (b) you may only continue to run existing installations of the Software, (c) you may not install the Software on any additional Hosts, and (d) any new installation of the Software shall require the purchase of a new license subscription from GoReplay LLC.
8. Disclaimer of Warranties. The Software is provided "as is," with all faults, defects and errors, and without warranty of any kind. Replay Software does not warrant that the Software will be free of bugs, errors, viruses or other defects, and Replay Software shall have no liability of any kind for the use of or inability to use the Software, the Software content or any associated service, and you acknowledge that it is not technically practicable for Replay Software to do so.
To the maximum extent permitted by applicable law, Replay Software disclaims all warranties, express, implied, arising by law or otherwise, regarding the Software, the Software content and their respective performance or suitability for your intended use, including without limitation any implied warranty of merchantability, fitness for a particular purpose.
8. Disclaimer of Warranties. The Software is provided "as is," with all faults, defects and errors, and without warranty of any kind. GoReplay LLC does not warrant that the Software will be free of bugs, errors, viruses or other defects, and GoReplay LLC shall have no liability of any kind for the use of or inability to use the Software, the Software content or any associated service, and you acknowledge that it is not technically practicable for GoReplay LLC to do so.
To the maximum extent permitted by applicable law, GoReplay LLC disclaims all warranties, express, implied, arising by law or otherwise, regarding the Software, the Software content and their respective performance or suitability for your intended use, including without limitation any implied warranty of merchantability, fitness for a particular purpose.
9. Limitation of Liability.
In no event will Replay Software be liable for any direct, indirect, consequential, incidental, special, exemplary, or punitive damages or liabilities whatsoever arising from or relating to the Software, the Software content or this Agreement, whether based on contract, tort (including negligence), strict liability or other theory, even if Replay Software has been advised of the possibility of such damages.
In no event will GoReplay LLC be liable for any direct, indirect, consequential, incidental, special, exemplary, or punitive damages or liabilities whatsoever arising from or relating to the Software, the Software content or this Agreement, whether based on contract, tort (including negligence), strict liability or other theory, even if GoReplay LLC has been advised of the possibility of such damages.
In no event will Replay Software' liability exceed the Software license price as indicated in the invoice. The existence of more than one claim will not enlarge or extend this limit.
In no event will GoReplay LLC liability exceed the Software license price as indicated in the invoice. The existence of more than one claim will not enlarge or extend this limit.
10. Remedies. Your exclusive remedy and Replay Software’ entire liability for breach of this Agreement shall be limited, at Replay Software’ sole and exclusive discretion, to (a) replacement of any defective software or documentation; or (b) refund of the license fee paid to Replay Software, payable in accordance with Replay Software' refund policy.
10. Remedies. Your exclusive remedy and GoReplay LLC’ entire liability for breach of this Agreement shall be limited, at GoReplay LLC’ sole and exclusive discretion, to (a) replacement of any defective software or documentation; or (b) refund of the license fee paid to GoReplay LLC, payable in accordance with GoReplay LLC' refund policy.
11. Acknowledgements.
11.1 Consent to the Use of Data. You agree that Replay Software and its affiliates may collect and use technical information gathered as part of the product support services. Replay Software may use this information solely to improve products and services and will not disclose this information in a form that personally identifies you.
11.1 Consent to the Use of Data. You agree that GoReplay LLC and its affiliates may collect and use technical information gathered as part of the product support services. GoReplay LLC may use this information solely to improve products and services and will not disclose this information in a form that personally identifies you.
11.2 Verification. We or a certified auditor acting on our behalf, may, upon its reasonable request and at its expense, audit you with respect to the use of the Software. Such audit may be conducted by mail, electronic means or through an in-person visit to your place of business. Any such in-person audit shall be conducted during regular business hours at your facilities and shall not unreasonably interfere with your business activities. We shall not remove, copy, or redistribute any electronic material during the course of an audit. If an audit reveals that you are using the Software in a way that is in material violation of the terms of the EULA, then you shall pay our reasonable costs of conducting the audit. In the case of a material violation, you agree to pay Us any amounts owing that are attributable to the unauthorized use. In the alternative, We reserve the right, at our sole option, to terminate the licenses for the Software.
@@ -74,13 +72,13 @@ In no event will Replay Software' liability exceed the Software license price as
13.1 Entire Agreement. This Agreement sets forth our entire agreement with respect to the Software and the subject matter hereof and supersedes all prior and contemporaneous understandings and agreements whether written or oral.
13.2 Amendment. Replay Software reserves the right, in its sole discretion, to amend this Agreement from time. Amendments to this Agreement can be located at: https://github.com/buger/gor/blob/master/COMM-LICENSE.
13.2 Amendment. GoReplay LLC reserves the right, in its sole discretion, to amend this Agreement from time. Amendments to this Agreement can be located at: https://github.com/buger/gor/blob/master/COMM-LICENSE.
13.3 Assignment. You may not assign this Agreement or any of its rights under this Agreement without the prior written consent of Replay Software and any attempted assignment without such consent shall be void.
13.3 Assignment. You may not assign this Agreement or any of its rights under this Agreement without the prior written consent of GoReplay LLC and any attempted assignment without such consent shall be void.
13.4 Export Compliance. You agree to comply with all applicable laws and regulations, including laws, regulations, orders or other restrictions on export, re-export or redistribution of software.
13.5 Indemnification. You agree to defend, indemnify, and hold harmless Replay Software from and against any lawsuits, claims, losses, damages, fines and expenses (including attorneys' fees and costs) arising out of your use of the Software or breach of this Agreement.
13.5 Indemnification. You agree to defend, indemnify, and hold harmless GoReplay LLC from and against any lawsuits, claims, losses, damages, fines and expenses (including attorneys' fees and costs) arising out of your use of the Software or breach of this Agreement.
13.6 Governing Law. This Agreement is governed by the laws of the State of Oregon and the United States without regard to conflicts of laws provisions thereof, and without regard to the United Nations Convention on the International Sale of Goods or the Uniform Computer Information Transactions Act, as currently enacted by any jurisdiction or as may be codified or amended from time to time by any jurisdiction. The jurisdiction and venue for actions related to the subject matter hereof shall be the state of Oregon and United States federal courts located in Portland, Oregon, and both parties hereby submit to the personal jurisdiction of such courts.
@@ -92,4 +90,4 @@ In no event will Replay Software' liability exceed the Software license price as
13.10 Headings. The headings of sections and paragraphs of this Agreement are for convenience of reference only and are not intended to restrict, affect or be of any weight in the interpretation or construction of the provisions of such sections or paragraphs.
14. Contact Information. If you have any questions about this EULA, or if you want to contact Replay Software for any reason, please direct correspondence to info@gortool.com
14. Contact Information. If you have any questions about this EULA, or if you want to contact GoReplay LLC for any reason, please direct correspondence to info@gortool.com
Dockerfile
+5 -3
View File
@@ -16,8 +16,10 @@ RUN go get github.com/google/gopacket
RUN go get -u github.com/golang/lint/golint
RUN go get -u github.com/aws/aws-sdk-go
WORKDIR /go/src/github.com/buger/gor/
ADD . /go/src/github.com/buger/gor/
WORKDIR /go/src/github.com/buger/gor-pro/
ADD . /go/src/github.com/buger/gor-pro/
RUN javac -cp /tmp/commons-io-2.4/commons-io-2.4.jar ./examples/middleware/echo.java
RUN wget http://archive.apache.org/dist/commons/io/binaries/commons-io-2.4-bin.tar.gz && tar xzf commons-io-2.4-bin.tar.gz && cd commons-io-2.4 && mv commons-io-2.4.jar /tmp/
RUN wget http://archive.apache.org/dist/commons/codec/binaries/commons-codec-1.9-bin.tar.gz && tar xzf commons-codec-1.9-bin.tar.gz
RUN javac -cp commons-io-2.4/commons-io-2.4.jar -cp commons-codec-1.9/commons-codec-1.9.jar ./examples/middleware/echo.java
RUN go get
LICENSE.txt
+15 -3
View File
@@ -1,7 +1,19 @@
Gor is an Open Source project licensed under the terms of
GoReplay is an Open Source project licensed under the terms of
the LGPLv3 license. Please see <http://www.gnu.org/licenses/lgpl-3.0.html>
for license text.
Gor Pro has a commercial-friendly license allowing private forks
and modifications of Gor. Please see http://gortool.com/#pro for
As a special exception to the GNU Lesser General Public License version 3
("LGPL3"), the copyright holders of this Library give you permission to
convey to a third party a Combined Work that links statically or dynamically
to this Library without providing any Minimal Corresponding Source or
Minimal Application Code as set out in 4d or providing the installation
information set out in section 4e, provided that you comply with the other
provisions of LGPL3 and provided that you meet, for the Application the
terms and conditions of the license(s) which apply to the Application.
TLDR: You are free to use Gor subpackages like `byteutils` or `proto` in your commercial projects.
GoReplay Pro has a commercial-friendly license allowing private forks
and modifications of GoReplay. Please see https://goreplay.org/pro.html for
more detail. You can find the commercial license terms in COMM-LICENSE.
Makefile
+14 -10
View File
@@ -1,28 +1,29 @@
SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go test_input.go elasticsearch.go http_modifier.go http_modifier_settings.go http_client.go middleware.go protocol.go output_file_settings.go output_s3.go
SOURCE_PATH = /go/src/github.com/buger/gor/
SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_null.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go test_input.go elasticsearch.go http_modifier.go http_modifier_settings.go http_client.go middleware.go protocol.go output_file_settings.go tcp_client.go output_binary.go
SOURCE_PATH = /go/src/github.com/buger/gor-pro/
PORT = 8000
FADDR = :8000
RUN = docker run -v `pwd`:$(SOURCE_PATH) -e AWS_ACCESS_KEY_ID=AKIAIOJPOSYCQCWU4YHQ -e AWS_SECRET_ACCESS_KEY=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 -p 0.0.0.0:$(PORT):$(PORT) -t -i gor
CONTAINER=gor-pro
RUN = docker run -v `pwd`:$(SOURCE_PATH) -e AWS_ACCESS_KEY_ID=AKIAIOJPOSYCQCWU4YHQ -e AWS_SECRET_ACCESS_KEY=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 -p 0.0.0.0:$(PORT):$(PORT) -t -i $(CONTAINER)
BENCHMARK = BenchmarkRAWInput
TEST = TestRawListenerBench
VERSION = DEV-$(shell date +%s)
LDFLAGS = -ldflags "-X main.VERSION=$(VERSION) -extldflags \"-static\""
MAC_LDFLAGS = -ldflags "-X main.VERSION=$(VERSION)"
LDFLAGS = -ldflags "-X main.VERSION=$(VERSION)_PRO -extldflags \"-static\""
MAC_LDFLAGS = -ldflags "-X main.VERSION=$(VERSION)_PRO"
FADDR = ":8000"
release: release-x64 release-mac
release-x64:
docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64 -i gor go build $(LDFLAGS) && tar -czf gor_$(VERSION)_x64.tar.gz gor && rm gor
docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64 -i $(CONTAINER) go build -o gor -tags netgo $(LDFLAGS) && tar -czf gor_$(VERSION)_PRO_x64.tar.gz gor && rm gor
release-x86:
docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 -i gor go build $(LDFLAGS) && tar -czf gor_$(VERSION)_x86.tar.gz gor && rm gor
docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 -i $(CONTAINER) go build -o gor -tags netgo $(LDFLAGS) && tar -czf gor_$(VERSION)_PRO_x86.tar.gz gor && rm gor
release-mac:
go build $(MAC_LDFLAGS) && tar -czf gor_$(VERSION)_mac.tar.gz gor && rm gor
go build -o gor $(MAC_LDFLAGS) && tar -czf gor_$(VERSION)_PRO_mac.tar.gz gor && rm gor
build:
docker build -t gor .
docker build -t $(CONTAINER) .
profile:
@@ -62,7 +63,7 @@ profile_test:
# Used mainly for debugging, because docker container do not have access to parent machine ports
run:
$(RUN) go run $(LDFLAGS) $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-raw-track-response --input-raw 127.0.0.1:9000 --input-http 127.0.0.1:9000 --verbose --debug --middleware "./examples/middleware/echo.sh"
$(RUN) go run $(LDFLAGS) $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-raw-track-response --input-raw 127.0.0.1:9000 --verbose --debug --middleware "./examples/middleware/echo.sh" --output-file requests.gor
run-2:
sudo -E go run $(SOURCE) --input-dummy="" --output-tcp localhost:27001 --verbose --debug
@@ -76,6 +77,9 @@ run-arg:
file-server:
go run $(SOURCE) file-server $(FADDR)
readpcap:
go run $(SOURCE) --input-raw $(FILE) --input-raw-engine pcap_file --output-null
record:
$(RUN) go run $(SOURCE) --input-dummy=0 --output-file=requests.gor --verbose --debug
README.md
+15 -15
View File
@@ -2,17 +2,24 @@
![Go Replay](http://i.imgur.com/ZG2ki5n.png)
## https://goreplay.org/
## About
Gor is an open-source tool for capturing and replaying live HTTP traffic into a test environment in order to continuously test your system with real data. It can be used to increase confidence in code deployments, configuration changes and infrastructure changes.
GoReplay is the simplest and safest way to test your app using real traffic before you put it into production.
As your application grows, the effort required to test it also grows exponentially. GoReplay offers you the simple idea of reusing your existing traffic for testing, which makes it incredibly powerful. Our state of art technique allows to analyze and record your application traffic without affecting it. This eliminates the risks that come with putting a third party component in the critical path.
GoReplay increases your confidence in code deployments, configuration changes and infrastructure changes. Did we mention that no coding is required?
Now you can test your code on real user sessions in an automated and repeatable fashion.
**No more falling down in production!**
Here is basic workflow: The listener server catches http traffic and sends it to the replay server or saves to file. The replay server forwards traffic to a given address.
![Diagram](http://i.imgur.com/9mqj2SK.png)
Check [latest documentation](http://github.com/buger/gor/wiki).
## Installation
Download latest binary from https://github.com/buger/gor/releases or [compile by yourself](https://github.com/buger/gor/wiki/Compilation).
@@ -21,14 +28,15 @@ Download latest binary from https://github.com/buger/gor/releases or [compile by
The most basic setup will be `sudo ./gor --input-raw :8000 --output-stdout` which acts like tcpdump.
If you already have test environment you can start replaying: `sudo ./gor --input-raw :8000 --output-http http://staging.env`.
See the our wiki and especially [Getting started](https://github.com/buger/gor/wiki/Getting-Started) wiki page for more info.
See the our [documentation](https://github.com/buger/gor/wiki/) and [Getting started](https://github.com/buger/gor/wiki/Getting-Started) page for more info.
## Newsletter
Subscribe to our [newsletter](https://www.getdrip.com/forms/89690474/submissions/new) to stay informed about the latest features and changes to Gor project.
## Want to Upgrade?
I also sell Gor Pro, extensions to Gor which provide more features, a commercial-friendly license and allow you to support high quality open source development all at the same time. Please see the Gor [homepage](https://gortool.com/) for more detail.
We have created a [GoReplay PRO](https://goreplay.org/pro.html) extension which provides additional features such as support for binary protocols like Thrift or ProtocolBuffers, saving and replaying from cloud storage, TCP sessions replication, etc. The PRO version also includes a commercial-friendly license, dedicated support, and it also allows you to support high-quality open source development.
Subscribe to the [quarterly newsletter](https://tinyletter.com/gor) to stay informed about the latest features and changes to Gor and its bigger siblings.
## Problems?
If you have a problem, please review the [FAQ](https://github.com/buger/gor/wiki/FAQ) and [Troubleshooting](https://github.com/buger/gor/wiki/Troubleshooting) wiki pages. Searching the [issues](https://github.com/buger/gor/issues) for your problem is also a good idea.
@@ -36,14 +44,6 @@ If you have a problem, please review the [FAQ](https://github.com/buger/gor/wiki
All bug-reports and suggestions should go though Github Issues or our [Google Group](https://groups.google.com/forum/#!forum/gor-users) (you can just send email to gor-users@googlegroups.com).
If you have a private question feel free to send email to support@gortool.com.
Useful resources:
* Product documentation is in the [wiki](http://github.com/buger/gor/wiki).
* Release announcements are made to the [@buger](http://twitter.com/buger) Twitter account and our [newsleter](https://tinyletter.com/gor)
If you need commercial support read more about Pro and Enterprise versions at our site [https://gortool.com/](https://gortool.com/)
## Contributing
@@ -66,7 +66,7 @@ If you need commercial support read more about Pro and Enterprise versions at ou
* [Granify](http://granify.com) - AI backed SaaS solution that enables online retailers to maximise their sales
* And many more!
If you are using Gor we are happy add you to the list and share your story, just write to: hello@gortool.com
If you are using Gor we are happy add you to the list and share your story, just write to: hello@goreplay.org
## Author
elasticsearch.go
+56 -53
View File
@@ -2,9 +2,8 @@ package main
import (
"encoding/json"
"github.com/buger/elastigo/api"
"github.com/buger/elastigo/core"
"github.com/buger/gor/proto"
"github.com/mattbaird/elastigo/lib"
"log"
"regexp"
"time"
@@ -19,33 +18,34 @@ func (e *ESUriErorr) Error() string {
type ESPlugin struct {
Active bool
ApiPort string
eConn *elastigo.Conn
Host string
Index string
indexor *core.BulkIndexer
indexor *elastigo.BulkIndexer
done chan bool
}
type ESRequestResponse struct {
ReqUrl []byte `json:"Req_URL"`
ReqMethod []byte `json:"Req_Method"`
ReqUserAgent []byte `json:"Req_User-Agent"`
ReqAcceptLanguage []byte `json:"Req_Accept-Language,omitempty"`
ReqAccept []byte `json:"Req_Accept,omitempty"`
ReqAcceptEncoding []byte `json:"Req_Accept-Encoding,omitempty"`
ReqIfModifiedSince []byte `json:"Req_If-Modified-Since,omitempty"`
ReqConnection []byte `json:"Req_Connection,omitempty"`
ReqCookies []byte `json:"Req_Cookies,omitempty"`
RespStatus []byte `json:"Resp_Status"`
RespStatusCode []byte `json:"Resp_Status-Code"`
RespProto []byte `json:"Resp_Proto,omitempty"`
RespContentLength []byte `json:"Resp_Content-Length,omitempty"`
RespContentType []byte `json:"Resp_Content-Type,omitempty"`
RespTransferEncoding []byte `json:"Resp_Transfer-Encoding,omitempty"`
RespContentEncoding []byte `json:"Resp_Content-Encoding,omitempty"`
RespExpires []byte `json:"Resp_Expires,omitempty"`
RespCacheControl []byte `json:"Resp_Cache-Control,omitempty"`
RespVary []byte `json:"Resp_Vary,omitempty"`
RespSetCookie []byte `json:"Resp_Set-Cookie,omitempty"`
ReqURL string `json:"Req_URL"`
ReqMethod string `json:"Req_Method"`
ReqUserAgent string `json:"Req_User-Agent"`
ReqAcceptLanguage string `json:"Req_Accept-Language,omitempty"`
ReqAccept string `json:"Req_Accept,omitempty"`
ReqAcceptEncoding string `json:"Req_Accept-Encoding,omitempty"`
ReqIfModifiedSince string `json:"Req_If-Modified-Since,omitempty"`
ReqConnection string `json:"Req_Connection,omitempty"`
ReqCookies string `json:"Req_Cookies,omitempty"`
RespStatus string `json:"Resp_Status"`
RespStatusCode string `json:"Resp_Status-Code"`
RespProto string `json:"Resp_Proto,omitempty"`
RespContentLength string `json:"Resp_Content-Length,omitempty"`
RespContentType string `json:"Resp_Content-Type,omitempty"`
RespTransferEncoding string `json:"Resp_Transfer-Encoding,omitempty"`
RespContentEncoding string `json:"Resp_Content-Encoding,omitempty"`
RespExpires string `json:"Resp_Expires,omitempty"`
RespCacheControl string `json:"Resp_Cache-Control,omitempty"`
RespVary string `json:"Resp_Vary,omitempty"`
RespSetCookie string `json:"Resp_Set-Cookie,omitempty"`
Rtt int64 `json:"RTT"`
Timestamp time.Time
}
@@ -76,24 +76,26 @@ func (p *ESPlugin) Init(URI string) {
if err != nil {
log.Fatal("Can't initialize ElasticSearch plugin.", err)
}
p.eConn = elastigo.NewConn()
p.eConn.SetPort(p.ApiPort)
p.eConn.SetHosts([]string{p.Host})
api.Domain = p.Host
api.Port = p.ApiPort
p.indexor = core.NewBulkIndexerErrors(50, 60)
p.indexor = p.eConn.NewBulkIndexerErrors(50, 60)
p.done = make(chan bool)
p.indexor.Run(p.done)
p.indexor.Start()
// Only start the ErrorHandler goroutine when in verbose mode
// no need to burn ressources otherwise
// go p.ErrorHandler()
if Settings.verbose {
// Only start the ErrorHandler goroutine when in verbose mode
// no need to burn ressources otherwise
go p.ErrorHandler()
}
log.Println("Initialized Elasticsearch Plugin")
return
}
func (p *ESPlugin) IndexerShutdown() {
p.done <- true
p.indexor.Stop()
return
}
@@ -118,28 +120,29 @@ func (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) {
}
t := time.Now()
rtt := p.RttDurationToMs(stop.Sub(start))
req = payloadBody(req)
esResp := ESRequestResponse{
ReqUrl: proto.Path(req),
ReqMethod: proto.Method(req),
ReqUserAgent: proto.Header(req, []byte("User-Agent")),
ReqAcceptLanguage: proto.Header(req, []byte("Accept-Language")),
ReqAccept: proto.Header(req, []byte("Accept")),
ReqAcceptEncoding: proto.Header(req, []byte("Accept-Encoding")),
ReqIfModifiedSince: proto.Header(req, []byte("If-Modified-Since")),
ReqConnection: proto.Header(req, []byte("Connection")),
ReqCookies: proto.Header(req, []byte("Cookie")),
RespStatus: proto.Status(resp),
RespStatusCode: proto.Status(resp),
RespProto: proto.Method(resp),
RespContentLength: proto.Header(resp, []byte("Content-Length")),
RespContentType: proto.Header(resp, []byte("Content-Type")),
RespTransferEncoding: proto.Header(resp, []byte("Transfer-Encoding")),
RespContentEncoding: proto.Header(resp, []byte("Content-Encoding")),
RespExpires: proto.Header(resp, []byte("Expires")),
RespCacheControl: proto.Header(resp, []byte("Cache-Control")),
RespVary: proto.Header(resp, []byte("Vary")),
RespSetCookie: proto.Header(resp, []byte("Set-Cookie")),
ReqURL: string(proto.Path(req)),
ReqMethod: string(proto.Method(req)),
ReqUserAgent: string(proto.Header(req, []byte("User-Agent"))),
ReqAcceptLanguage: string(proto.Header(req, []byte("Accept-Language"))),
ReqAccept: string(proto.Header(req, []byte("Accept"))),
ReqAcceptEncoding: string(proto.Header(req, []byte("Accept-Encoding"))),
ReqIfModifiedSince: string(proto.Header(req, []byte("If-Modified-Since"))),
ReqConnection: string(proto.Header(req, []byte("Connection"))),
ReqCookies: string(proto.Header(req, []byte("Cookie"))),
RespStatus: string(proto.Status(resp)),
RespStatusCode: string(proto.Status(resp)),
RespProto: string(proto.Method(resp)),
RespContentLength: string(proto.Header(resp, []byte("Content-Length"))),
RespContentType: string(proto.Header(resp, []byte("Content-Type"))),
RespTransferEncoding: string(proto.Header(resp, []byte("Transfer-Encoding"))),
RespContentEncoding: string(proto.Header(resp, []byte("Content-Encoding"))),
RespExpires: string(proto.Header(resp, []byte("Expires"))),
RespCacheControl: string(proto.Header(resp, []byte("Cache-Control"))),
RespVary: string(proto.Header(resp, []byte("Vary"))),
RespSetCookie: string(proto.Header(resp, []byte("Set-Cookie"))),
Rtt: rtt,
Timestamp: t,
}
@@ -147,7 +150,7 @@ func (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) {
if err != nil {
log.Println(err)
} else {
p.indexor.Index(p.Index, "RequestResponse", "", "", &t, j, true)
p.indexor.Index(p.Index, "RequestResponse", "", "", "", &t, j)
}
return
}
emitter.go
+18 -13
View File
@@ -4,6 +4,7 @@ import (
"bytes"
"io"
"time"
"hash/fnv"
)
// Start initialize loop for sending data from inputs to outputs
@@ -15,7 +16,7 @@ func Start(stop chan int) {
middleware.ReadFrom(in)
}
// We going only to read responses, so using same ReadFrom method
// We are going only to read responses, so using same ReadFrom method
for _, out := range Plugins.Outputs {
if r, ok := out.(io.Reader); ok {
middleware.ReadFrom(r)
@@ -32,13 +33,7 @@ func Start(stop chan int) {
for {
select {
case <-stop:
pluginMu.Lock()
for _, p := range Plugins.All {
if cp, ok := p.(io.Closer); ok {
cp.Close()
}
}
pluginMu.Unlock()
finalize()
return
case <-time.After(100 * time.Millisecond):
}
@@ -87,13 +82,23 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) {
}
if Settings.splitOutput {
// Simple round robin
writers[wIndex].Write(payload)
if Settings.recognizeTCPSessions {
hasher := fnv.New32a()
// First 20 bytes contain tcp session
id := payloadID(payload)
hasher.Write(id[:20])
wIndex++
wIndex = int(hasher.Sum32()) % len(writers)
writers[wIndex].Write(payload)
} else {
// Simple round robin
writers[wIndex].Write(payload)
if wIndex >= len(writers) {
wIndex = 0
wIndex++
if wIndex >= len(writers) {
wIndex = 0
}
}
} else {
for _, dst := range writers {
emitter_test.go
+68 -1
View File
@@ -5,6 +5,7 @@ import (
"sync"
"sync/atomic"
"testing"
"bytes"
)
func TestEmitter(t *testing.T) {
@@ -31,7 +32,7 @@ func TestEmitter(t *testing.T) {
close(quit)
}
func TestEmitterRoundRobin(t *testing.T) {
func TestEmitterSplitRoundRobin(t *testing.T) {
wg := new(sync.WaitGroup)
quit := make(chan int)
@@ -72,6 +73,72 @@ func TestEmitterRoundRobin(t *testing.T) {
Settings.splitOutput = false
}
func TestEmitterSplitSession(t *testing.T) {
wg1 := new(sync.WaitGroup)
wg2 := new(sync.WaitGroup)
wg1.Add(1000)
wg2.Add(1000)
// Base uuids, only 1 letter changed
uuid1 := []byte("1234567890123456789a0000")
uuid2 := []byte("1234567890123456789d0000")
quit := make(chan int)
input := NewTestInput()
input.disableHeaders = true
var counter1, counter2 int32
output1 := NewTestOutput(func(data []byte) {
atomic.AddInt32(&counter1, 1)
if !bytes.Equal(uuid1[:20], payloadID(data)[:20]) {
t.Errorf("All tcp sessions should have same id")
}
wg1.Done()
})
output2 := NewTestOutput(func(data []byte) {
atomic.AddInt32(&counter2, 1)
if !bytes.Equal(uuid2[:20], payloadID(data)[:20]) {
t.Errorf("All tcp sessions should have same id")
}
wg2.Done()
})
Plugins.Inputs = []io.Reader{input}
Plugins.Outputs = []io.Writer{output1, output2}
Settings.splitOutput = true
Settings.recognizeTCPSessions = true
go Start(quit)
for i := 0; i < 1000; i++ {
// Keep session but randomize ACK
copy(uuid1[20:], randByte(4))
input.EmitBytes([]byte("1 " + string(uuid1) + " 1\n" + "GET / HTTP/1.1\r\n\r\n"))
}
for i := 0; i < 1000; i++ {
// Keep session but randomize ACK
copy(uuid2[20:], randByte(4))
input.EmitBytes([]byte("1 " + string(uuid2) + " 1\n" + "GET / HTTP/1.1\r\n\r\n"))
}
wg1.Wait()
wg2.Wait()
close(quit)
if counter1 != 1000 || counter2 != 1000 {
t.Errorf("Round robin should split traffic equally: %d vs %d", counter1, counter2)
}
Settings.splitOutput = false
Settings.recognizeTCPSessions = false
}
func BenchmarkEmitter(b *testing.B) {
wg := new(sync.WaitGroup)
quit := make(chan int)
examples/middleware/echo.clj
+53
View File
@@ -0,0 +1,53 @@
(ns echo.core
(:gen-class)
(:require [clojure.string :as cs]
[clojure.java.io :as io])
(:import org.apache.commons.codec.binary.Hex
java.io.BufferedReader
java.io.IOException
java.io.InputStreamReader))
(defn transform-http-msg
"Function that transforms/filters the incoming HTTP messages."
[headers body]
;; do actual transformations here
[headers body])
(defn decode-hex-string
"Decode an Hex-encoded string."
[s]
(String. (Hex/decodeHex (.toCharArray s))))
(defn encode-hex-string
"Encode a string to a hex-encoded string."
[^String s]
(String. (Hex/encodeHex (.getBytes s))))
(defn -main
[& args]
(let [br (BufferedReader. (InputStreamReader. System/in))]
(try
(loop [hex-line (.readLine br)]
(let [decoded-req (decode-hex-string hex-line)
;; empty line separates headers from body
http-request (partition-by empty? (cs/split-lines decoded-req))
headers (first http-request)
;; HTTP messages can contain no body:
body (when (= 3 (count http-request)) (last http-request))
[new-headers new-body] (transform-http-msg headers body)]
(println (encode-hex-string (str (cs/join "\n" headers)
(when body
(str "\n\n"
(cs/join "\n" body)))))))
(when-let [line (.readLine br)]
(recur line)))
(catch IOException e nil))))
examples/middleware/echo.java
+26 -5
View File
@@ -2,8 +2,25 @@ import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
public class echo {
public static void main(String[] args) {
import org.apache.commons.codec.DecoderException;
import org.apache.commons.codec.binary.Hex;
class Echo {
public static String decodeHexString(String s) throws DecoderException {
return new String(Hex.decodeHex(s.toCharArray()));
}
public static String encodeHexString(String s) {
return new String(Hex.encodeHex(s.getBytes()));
}
public static String transformHTTPMessage(String req) {
// do actual transformations here
return req;
}
public static void main(String[] args) throws DecoderException {
if(args != null){
for(String arg : args){
System.out.println(arg);
@@ -12,16 +29,20 @@ public class echo {
}
BufferedReader stdin = new BufferedReader(new InputStreamReader(
System.in));
System.in));
String line = null;
try {
while ((line = stdin.readLine()) != null) {
String decodedLine = decodeHexString(line);
System.out.println(line);
String transformedLine = transformHTTPMessage(decodedLine);
String encodedLine = encodeHexString(transformedLine);
System.out.println(encodedLine);
}
} catch (IOException e) {
}
}
}
}
examples/middleware/echo.sh
+4
View File
@@ -2,6 +2,10 @@
#
# `xxd` utility included into vim-common package
# It allow hex decoding/encoding
#
# This example may broke if you request contains `null` string, you may consider using pipes instead.
# See: https://github.com/buger/gor/issues/309
#
function log {
# Logging to stderr, because stdout/stdin used for data transfer
gor.go
+22 -8
View File
@@ -78,17 +78,31 @@ func main() {
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
go func() {
<-c
for _, p := range Plugins.All {
if cp, ok := p.(io.Closer); ok {
cp.Close()
}
}
finalize()
os.Exit(1)
}()
Start(nil)
if Settings.exitAfter > 0 {
log.Println("Running gor for a duration of", Settings.exitAfter)
closeCh := make(chan int)
time.AfterFunc(Settings.exitAfter, func() {
log.Println("Stopping gor after", Settings.exitAfter)
close(closeCh)
})
Start(closeCh)
} else {
Start(nil)
}
}
func finalize() {
for _, p := range Plugins.All {
if cp, ok := p.(io.Closer); ok {
cp.Close()
}
}
}
func profileCPU(cpuprofile string) {
http_client.go
+29 -14
View File
@@ -3,6 +3,7 @@ package main
import (
"bytes"
"crypto/tls"
"encoding/base64"
"io"
"log"
"net"
@@ -44,6 +45,7 @@ type HTTPClient struct {
baseURL string
scheme string
host string
auth string
conn net.Conn
respBuf []byte
config *HTTPClientConfig
@@ -56,17 +58,12 @@ func NewHTTPClient(baseURL string, config *HTTPClientConfig) *HTTPClient {
}
u, _ := url.Parse(baseURL)
if !strings.Contains(u.Host, ":") {
if u.Scheme != "http" {
u.Host += ":" + defaultPorts[u.Scheme]
}
}
if config.Timeout.Nanoseconds() == 0 {
config.Timeout = 5 * time.Second
}
if config.Timeout == 0 {
config.Timeout = time.Second
}
config.ConnectionTimeout = time.Second
config.ConnectionTimeout = config.Timeout
if config.ResponseBufferSize == 0 {
config.ResponseBufferSize = 100 * 1024 // 100kb
@@ -79,6 +76,10 @@ func NewHTTPClient(baseURL string, config *HTTPClientConfig) *HTTPClient {
client.respBuf = make([]byte, config.ResponseBufferSize)
client.config = config
if u.User != nil {
client.auth = "Basic " + base64.StdEncoding.EncodeToString([]byte(u.User.String()))
}
return client
}
@@ -86,13 +87,13 @@ func (c *HTTPClient) Connect() (err error) {
c.Disconnect()
if !strings.Contains(c.host, ":") {
c.conn, err = net.DialTimeout("tcp", c.host+":80", c.config.ConnectionTimeout)
c.conn, err = net.DialTimeout("tcp", c.host + ":" + defaultPorts[c.scheme], c.config.ConnectionTimeout)
} else {
c.conn, err = net.DialTimeout("tcp", c.host, c.config.ConnectionTimeout)
}
if c.scheme == "https" {
tlsConn := tls.Client(c.conn, &tls.Config{InsecureSkipVerify: true})
tlsConn := tls.Client(c.conn, &tls.Config{InsecureSkipVerify: true, ServerName: c.host})
if err = tlsConn.Handshake(); err != nil {
return
@@ -164,6 +165,10 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
data = proto.SetHost(data, []byte(c.baseURL), []byte(c.host))
}
if c.auth != "" {
data = proto.SetHeader(data, []byte("Authorization"), []byte(c.auth))
}
if c.config.Debug {
Debug("[HTTPClient] Sending:", string(data))
}
@@ -206,9 +211,14 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
if bytes.Equal(proto.Header(c.respBuf, []byte("Transfer-Encoding")), []byte("chunked")) {
chunked = true
} else {
l := proto.Header(c.respBuf, []byte("Content-Length"))
if len(l) > 0 {
contentLength, _ = strconv.Atoi(string(l))
status, _ := strconv.Atoi(string(proto.Status(c.respBuf)))
if (status >= 100 && status < 200) || status == 204 || status == 304 {
contentLength = 0
} else {
l := proto.Header(c.respBuf, []byte("Content-Length"))
if len(l) > 0 {
contentLength, _ = strconv.Atoi(string(l))
}
}
}
@@ -312,6 +322,11 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
}
}
if bytes.Equal(proto.Status(payload), []byte("400")) {
c.Disconnect()
Debug("[HTTPClient] Closed connection on 400 response")
}
c.redirectsCount = 0
return payload, err
http_client_test.go
+73 -2
View File
@@ -11,6 +11,7 @@ import (
"net/http/httptest"
"net/http/httputil"
_ "reflect"
"strings"
"sync"
"testing"
"time"
@@ -23,8 +24,8 @@ func TestHTTPClientURLPort(t *testing.T) {
}
c2 := NewHTTPClient("https://example.com", &HTTPClientConfig{})
if c2.baseURL != "https://example.com:443" {
t.Error("Sould add 443 port for https:", c2.baseURL)
if c2.baseURL != "https://example.com" {
t.Error("Sould not add 443 port for https:", c2.baseURL)
}
c3 := NewHTTPClient("https://example.com:1", &HTTPClientConfig{})
@@ -87,6 +88,37 @@ func TestHTTPClientSend(t *testing.T) {
wg.Wait()
}
func TestHTTPClientResonseByClose(t *testing.T) {
wg := new(sync.WaitGroup)
payload := []byte("GET / HTTP/1.1\r\n\r\n")
ln, _ := net.Listen("tcp", ":0")
go func() {
for {
conn, _ := ln.Accept()
buf := make([]byte, 4096)
conn.Read(buf)
conn.Write([]byte("HTTP/1.1 200 OK\r\n\r\n"))
conn.Write([]byte("ab"))
conn.Close()
wg.Done()
}
}()
client := NewHTTPClient(ln.Addr().String(), &HTTPClientConfig{Debug: true})
wg.Add(1)
resp, _ := client.Send(payload)
if !bytes.Equal(resp, []byte("HTTP/1.1 200 OK\r\n\r\nab")) {
t.Error("Should return valid response", string(resp))
}
wg.Wait()
}
// https://github.com/buger/gor/issues/184
func TestHTTPClientResponseBuffer(t *testing.T) {
testCases := []struct {
@@ -325,6 +357,45 @@ func TestHTTPClientRedirectLimit(t *testing.T) {
wg.Wait()
}
func TestHTTPClientBasicAuth(t *testing.T) {
wg := new(sync.WaitGroup)
wg.Add(2)
GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
user, pass, _ := r.BasicAuth()
if user != "user" || pass != "pass" {
http.Error(w, "Unauthorized.", 401)
wg.Done()
return
}
wg.Done()
}))
defer server.Close()
client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: false})
resp, _ := client.Send(GETPayload)
client.Disconnect()
if !bytes.Equal(proto.Status(resp), []byte("401")) {
t.Error("Should return unauthorized error", string(resp))
}
authUrl := strings.Replace(server.URL, "http://", "http://user:pass@", -1)
client = NewHTTPClient(authUrl, &HTTPClientConfig{Debug: false})
resp, _ = client.Send(GETPayload)
client.Disconnect()
if !bytes.Equal(proto.Status(resp), []byte("200")) {
t.Error("Should return proper response", string(resp))
}
wg.Wait()
}
func TestHTTPClientHandleHTTP10(t *testing.T) {
wg := new(sync.WaitGroup)
input_dummy.go
+2 -2
View File
@@ -34,10 +34,10 @@ func (i *DummyInput) emit() {
select {
case <-ticker.C:
uuid := uuid()
reqh := payloadHeader(RequestPayload, uuid, time.Now().UnixNano())
reqh := payloadHeader(RequestPayload, uuid, time.Now().UnixNano(), -1)
i.data <- append(reqh, []byte("GET / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
resh := payloadHeader(ResponsePayload, uuid, 1)
resh := payloadHeader(ResponsePayload, uuid, time.Now().UnixNano()+1, 1)
i.data <- append(resh, []byte("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")...)
}
}
input_http.go
+1 -1
View File
@@ -29,7 +29,7 @@ func NewHTTPInput(address string) (i *HTTPInput) {
func (i *HTTPInput) Read(data []byte) (int, error) {
buf := <-i.data
header := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano())
header := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
copy(data[0:len(header)], header)
copy(data[len(header):], buf)
input_raw.go
+25 -7
View File
@@ -1,8 +1,8 @@
package main
import (
"github.com/buger/gor/proto"
raw "github.com/buger/gor/raw_socket_listener"
"github.com/buger/gor-pro/proto"
raw "github.com/buger/gor-pro/raw_socket_listener"
"log"
"net"
"time"
@@ -18,16 +18,18 @@ type RAWInput struct {
realIPHeader []byte
trackResponse bool
listener *raw.Listener
protocol raw.TCPProtocol
}
// Available engines for intercepting traffic
const (
EngineRawSocket = 1 << iota
EnginePcap
EnginePcapFile
)
// NewRAWInput constructor for RAWInput. Accepts address with port as argument.
func NewRAWInput(address string, engine int, trackResponse bool, expire time.Duration, realIPHeader string) (i *RAWInput) {
func NewRAWInput(address string, engine int, trackResponse bool, expire time.Duration, realIPHeader string, protocol string) (i *RAWInput) {
i = new(RAWInput)
i.data = make(chan *raw.TCPMessage)
i.address = address
@@ -37,6 +39,15 @@ func NewRAWInput(address string, engine int, trackResponse bool, expire time.Dur
i.quit = make(chan bool)
i.trackResponse = trackResponse
switch protocol {
case "http":
i.protocol = raw.ProtocolHTTP
case "binary":
i.protocol = raw.ProtocolBinary
default:
log.Fatal("Unsupported protocol:", protocol)
}
i.listen(address)
i.listener.IsReady()
@@ -50,12 +61,12 @@ func (i *RAWInput) Read(data []byte) (int, error) {
var header []byte
if msg.IsIncoming {
header = payloadHeader(RequestPayload, msg.UUID(), msg.Start.UnixNano())
header = payloadHeader(RequestPayload, msg.UUID(), msg.Start.UnixNano(), -1)
if len(i.realIPHeader) > 0 {
buf = proto.SetHeader(buf, i.realIPHeader, []byte(msg.IP().String()))
}
} else {
header = payloadHeader(ResponsePayload, msg.UUID(), msg.End.UnixNano()-msg.AssocMessage.Start.UnixNano())
header = payloadHeader(ResponsePayload, msg.UUID(), msg.AssocMessage.Start.UnixNano(), msg.End.UnixNano()-msg.AssocMessage.Start.UnixNano())
}
copy(data[0:len(header)], header)
@@ -69,11 +80,17 @@ func (i *RAWInput) listen(address string) {
host, port, err := net.SplitHostPort(address)
if i.engine == EnginePcapFile {
host = address
port = "1"
err = nil
}
if err != nil {
log.Fatal("input-raw: error while parsing address", err)
}
i.listener = raw.NewListener(host, port, i.engine, i.trackResponse, i.expire)
i.listener = raw.NewListener(host, port, i.engine, i.trackResponse, i.expire, i.protocol)
ch := i.listener.Receiver()
@@ -97,7 +114,8 @@ func (i *RAWInput) String() string {
return "Intercepting traffic from: " + i.address
}
func (i *RAWInput) Close() {
func (i *RAWInput) Close() error {
i.listener.Close()
close(i.quit)
return nil
}
input_raw_test.go
+55 -6
View File
@@ -43,7 +43,7 @@ func TestRAWInputIPv4(t *testing.T) {
var respCounter, reqCounter int64
input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "X-Real-IP")
input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "X-Real-IP", "http")
defer input.Close()
output := NewTestOutput(func(data []byte) {
@@ -79,6 +79,55 @@ func TestRAWInputIPv4(t *testing.T) {
}
wg.Wait()
close(quit)
}
func TestRAWInputNoKeepAlive(t *testing.T) {
wg := new(sync.WaitGroup)
quit := make(chan int)
listener, err := net.Listen("tcp", ":0")
if err != nil {
t.Fatal(err)
}
origin := &http.Server{
Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("a"))
w.Write([]byte("b"))
}),
ReadTimeout: 10 * time.Second,
WriteTimeout: 10 * time.Second,
}
origin.SetKeepAlivesEnabled(false)
go origin.Serve(listener)
defer listener.Close()
originAddr := listener.Addr().String()
input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "", "http")
defer input.Close()
output := NewTestOutput(func(data []byte) {
wg.Done()
})
Plugins.Inputs = []io.Reader{input}
Plugins.Outputs = []io.Writer{output}
client := NewHTTPClient("http://"+listener.Addr().String(), &HTTPClientConfig{})
go Start(quit)
for i := 0; i < 100; i++ {
// request + response
wg.Add(2)
client.Get("/")
time.Sleep(2 * time.Millisecond)
}
wg.Wait()
close(quit)
}
@@ -102,7 +151,7 @@ func TestRAWInputIPv6(t *testing.T) {
var respCounter, reqCounter int64
input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "")
input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "", "http")
defer input.Close()
output := NewTestOutput(func(data []byte) {
@@ -153,7 +202,7 @@ func TestInputRAW100Expect(t *testing.T) {
originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
input := NewRAWInput(originAddr, EnginePcap, true, time.Second, "")
input := NewRAWInput(originAddr, EnginePcap, true, time.Second, "", "http")
defer input.Close()
// We will use it to get content of raw HTTP request
@@ -216,7 +265,7 @@ func TestInputRAWChunkedEncoding(t *testing.T) {
}))
originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
input := NewRAWInput(originAddr, EnginePcap, true, time.Second, "")
input := NewRAWInput(originAddr, EnginePcap, true, time.Second, "", "http")
defer input.Close()
replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -280,7 +329,7 @@ func TestInputRAWLargePayload(t *testing.T) {
}))
originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "")
input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "", "http")
defer input.Close()
replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
@@ -325,7 +374,7 @@ func BenchmarkRAWInput(b *testing.B) {
var respCounter, reqCounter int64
input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "")
input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "", "http")
defer input.Close()
output := NewTestOutput(func(data []byte) {
middleware_test.go
+2 -2
View File
@@ -117,7 +117,7 @@ func TestEchoMiddleware(t *testing.T) {
// Catch traffic from one service
fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
input := NewRAWInput(fromAddr, EnginePcap, true, testRawExpire, "")
input := NewRAWInput(fromAddr, EnginePcap, true, testRawExpire, "", "http")
defer input.Close()
// And redirect to another
@@ -179,7 +179,7 @@ func TestTokenMiddleware(t *testing.T) {
fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
// Catch traffic from one service
input := NewRAWInput(fromAddr, EnginePcap, true, testRawExpire, "")
input := NewRAWInput(fromAddr, EnginePcap, true, testRawExpire, "", "http")
defer input.Close()
// And redirect to another
output_binary.go
+175
View File
@@ -0,0 +1,175 @@
package main
import (
"io"
"sync/atomic"
"time"
)
// BinaryOutputConfig struct for holding binary output configuration
type BinaryOutputConfig struct {
workers int
Timeout time.Duration
BufferSize int
Debug bool
TrackResponses bool
}
// BinaryOutput plugin manage pool of workers which send request to replayed server
// By default workers pool is dynamic and starts with 10 workers
// You can specify fixed number of workers using `--output-tcp-workers`
type BinaryOutput struct {
// Keep this as first element of struct because it guarantees 64bit
// alignment. atomic.* functions crash on 32bit machines if operand is not
// aligned at 64bit. See https://github.com/golang/go/issues/599
activeWorkers int64
address string
queue chan []byte
responses chan response
needWorker chan int
config *BinaryOutputConfig
queueStats *GorStat
}
// NewBinaryOutput constructor for BinaryOutput
// Initialize workers
func NewBinaryOutput(address string, config *BinaryOutputConfig) io.Writer {
o := new(BinaryOutput)
o.address = address
o.config = config
o.queue = make(chan []byte, 1000)
o.responses = make(chan response, 1000)
o.needWorker = make(chan int, 1)
// Initial workers count
if o.config.workers == 0 {
o.needWorker <- initialDynamicWorkers
} else {
o.needWorker <- o.config.workers
}
if len(Settings.middleware) > 0 {
o.config.TrackResponses = true
}
go o.workerMaster()
return o
}
func (o *BinaryOutput) workerMaster() {
for {
newWorkers := <-o.needWorker
for i := 0; i < newWorkers; i++ {
go o.startWorker()
}
// Disable dynamic scaling if workers poll fixed size
if o.config.workers != 0 {
return
}
}
}
func (o *BinaryOutput) startWorker() {
client := NewTCPClient(o.address, &TCPClientConfig{
Debug: o.config.Debug,
Timeout: o.config.Timeout,
ResponseBufferSize: o.config.BufferSize,
})
deathCount := 0
atomic.AddInt64(&o.activeWorkers, 1)
for {
select {
case data := <-o.queue:
o.sendRequest(client, data)
deathCount = 0
case <-time.After(time.Millisecond * 100):
// When dynamic scaling enabled workers die after 2s of inactivity
if o.config.workers == 0 {
deathCount++
} else {
continue
}
if deathCount > 20 {
workersCount := atomic.LoadInt64(&o.activeWorkers)
// At least 1 startWorker should be alive
if workersCount != 1 {
atomic.AddInt64(&o.activeWorkers, -1)
return
}
}
}
}
}
func (o *BinaryOutput) Write(data []byte) (n int, err error) {
if !isRequestPayload(data) {
return len(data), nil
}
buf := make([]byte, len(data))
copy(buf, data)
o.queue <- buf
if o.config.workers == 0 {
workersCount := atomic.LoadInt64(&o.activeWorkers)
if len(o.queue) > int(workersCount) {
o.needWorker <- len(o.queue)
}
}
return len(data), nil
}
func (o *BinaryOutput) Read(data []byte) (int, error) {
resp := <-o.responses
Debug("[OUTPUT-TCP] Received response:", string(resp.payload))
header := payloadHeader(ReplayedResponsePayload, resp.uuid, resp.startedAt, resp.roundTripTime)
copy(data[0:len(header)], header)
copy(data[len(header):], resp.payload)
return len(resp.payload) + len(header), nil
}
func (o *BinaryOutput) sendRequest(client *TCPClient, request []byte) {
meta := payloadMeta(request)
if len(meta) < 2 {
return
}
uuid := meta[1]
body := payloadBody(request)
start := time.Now()
resp, err := client.Send(body)
stop := time.Now()
if err != nil {
Debug("Request error:", err)
}
if o.config.TrackResponses {
o.responses <- response{resp, uuid, start.UnixNano(), stop.UnixNano() - start.UnixNano()}
}
}
func (o *BinaryOutput) String() string {
return "TCP output: " + o.address
}
output_file.go
+24 -6
View File
@@ -8,9 +8,11 @@ import (
"log"
"os"
"path/filepath"
"runtime/debug"
"sort"
"strconv"
"strings"
"sync"
"time"
)
@@ -34,6 +36,7 @@ type FileOutputConfig struct {
// FileOutput output plugin
type FileOutput struct {
mu sync.Mutex
pathTemplate string
currentName string
file *os.File
@@ -92,7 +95,6 @@ func setFileIndex(name string, idx int) string {
withoutExt := strings.TrimSuffix(name, ext)
if i := strings.LastIndex(withoutExt, "_"); i != -1 {
// Only prefixes with numbers counts
if _, err := strconv.Atoi(withoutExt[i+1:]); err == nil {
withoutExt = withoutExt[:i]
}
@@ -128,6 +130,9 @@ func (s sortByFileIndex) Less(i, j int) bool {
}
func (o *FileOutput) filename() string {
defer o.mu.Unlock()
o.mu.Lock()
path := o.pathTemplate
for name, fn := range dateFileNameFuncs {
@@ -171,7 +176,7 @@ func (o *FileOutput) filename() string {
}
func (o *FileOutput) updateName() {
o.currentName = o.filename()
o.currentName = filepath.Clean(o.filename())
}
func (o *FileOutput) Write(data []byte) (n int, err error) {
@@ -180,6 +185,7 @@ func (o *FileOutput) Write(data []byte) (n int, err error) {
}
if o.file == nil || o.currentName != o.file.Name() {
o.mu.Lock()
o.Close()
o.file, err = os.OpenFile(o.currentName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
@@ -196,6 +202,7 @@ func (o *FileOutput) Write(data []byte) (n int, err error) {
}
o.queueLength = 0
o.mu.Unlock()
}
o.writer.Write(data)
@@ -207,16 +214,26 @@ func (o *FileOutput) Write(data []byte) (n int, err error) {
}
func (o *FileOutput) flush() {
// Don't exit on panic
defer func() {
if r := recover(); r != nil {
log.Println("PANIC while file flush: ", r, o, string(debug.Stack()))
}
}()
defer o.mu.Unlock()
o.mu.Lock()
if o.file != nil {
if strings.HasSuffix(o.currentName, ".gz") {
o.writer.(*gzip.Writer).Flush()
} else {
o.writer.(*bufio.Writer).Flush()
}
}
if stat, err := o.file.Stat(); err != nil {
o.chunkSize = int(stat.Size())
if stat, err := o.file.Stat(); err != nil {
o.chunkSize = int(stat.Size())
}
}
}
@@ -224,7 +241,7 @@ func (o *FileOutput) String() string {
return "File output: " + o.file.Name()
}
func (o *FileOutput) Close() {
func (o *FileOutput) Close() error {
if o.file != nil {
if strings.HasSuffix(o.currentName, ".gz") {
o.writer.(*gzip.Writer).Close()
@@ -237,4 +254,5 @@ func (o *FileOutput) Close() {
go o.config.onClose(o.file.Name())
}
}
return nil
}
output_file_test.go
+12
View File
@@ -53,6 +53,17 @@ func TestFileOutput(t *testing.T) {
close(quit)
}
func TestFileOutputWithNameCleaning(t *testing.T) {
output := &FileOutput{pathTemplate: "./test_requests.gor", config: &FileOutputConfig{flushInterval: time.Minute, append: false}}
expectedFileName := "test_requests_0.gor"
output.updateName()
if expectedFileName != output.currentName {
t.Errorf("Expected path %s but got %s", expectedFileName, output.currentName)
}
}
func TestFileOutputPathTemplate(t *testing.T) {
output := &FileOutput{pathTemplate: "/tmp/log-%Y-%m-%d-%S", config: &FileOutputConfig{flushInterval: time.Minute, append: true}}
now := time.Now()
@@ -166,6 +177,7 @@ func TestSetFileIndex(t *testing.T) {
{"/tmp/logs_1", 0, "/tmp/logs_0"},
{"/tmp/logs_0", 10, "/tmp/logs_10"},
{"/tmp/logs_0.gz", 10, "/tmp/logs_10.gz"},
{"/tmp/logs_underscores.gz", 10, "/tmp/logs_underscores_10.gz"},
}
for _, c := range tests {
output_http.go
+95 -21
View File
@@ -4,16 +4,59 @@ import (
"io"
"sync/atomic"
"time"
"fmt"
"github.com/buger/gor/proto"
"github.com/buger/gor-pro/proto"
)
var _ = fmt.Println
const initialDynamicWorkers = 10
type httpWorker struct {
output *HTTPOutput
client *HTTPClient
lastActivity time.Time
queue chan []byte
stop chan bool
}
func newHTTPWorker(output *HTTPOutput, queue chan []byte) *httpWorker {
client := NewHTTPClient(output.address, &HTTPClientConfig{
FollowRedirects: output.config.redirectLimit,
Debug: output.config.Debug,
OriginalHost: output.config.OriginalHost,
Timeout: output.config.Timeout,
ResponseBufferSize: output.config.BufferSize,
})
w := &httpWorker{client: client}
if queue == nil {
w.queue = make(chan []byte, 100)
} else {
w.queue = queue
}
w.stop = make(chan bool)
go func(){
for {
select {
case payload := <-w.queue:
output.sendRequest(client, payload)
case <- w.stop:
return
}
}
}()
return w
}
type response struct {
payload []byte
uuid []byte
roundTripTime int64
startedAt int64
}
// HTTPOutputConfig struct for holding http output configuration
@@ -43,6 +86,8 @@ type HTTPOutput struct {
// aligned at 64bit. See https://github.com/golang/go/issues/599
activeWorkers int64
workerSessions map[string]*httpWorker
address string
limit int
queue chan []byte
@@ -90,7 +135,12 @@ func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer {
o.config.TrackResponses = true
}
go o.workerMaster()
if Settings.recognizeTCPSessions {
o.workerSessions = make(map[string]*httpWorker, 100)
go o.sessionWorkerMaster()
} else {
go o.workerMaster()
}
return o
}
@@ -98,6 +148,7 @@ func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer {
func (o *HTTPOutput) workerMaster() {
for {
newWorkers := <-o.needWorker
atomic.AddInt64(&o.activeWorkers, int64(newWorkers))
for i := 0; i < newWorkers; i++ {
go o.startWorker()
}
@@ -109,6 +160,38 @@ func (o *HTTPOutput) workerMaster() {
}
}
func (o *HTTPOutput) sessionWorkerMaster() {
gc := time.Tick(time.Second)
for {
select {
case p := <-o.queue:
id := payloadID(p)
sessionID := string(id[0:20])
worker, ok := o.workerSessions[sessionID]
if !ok {
atomic.AddInt64(&o.activeWorkers, 1)
worker = newHTTPWorker(o, nil)
o.workerSessions[sessionID] = worker
}
worker.queue <- p
worker.lastActivity = time.Now()
case <-gc:
now := time.Now()
for id, w := range o.workerSessions {
if !w.lastActivity.IsZero() && now.Sub(w.lastActivity) >= 60 * time.Second {
w.stop <- true
delete(o.workerSessions, id)
atomic.AddInt64(&o.activeWorkers, -1)
}
}
}
}
}
func (o *HTTPOutput) startWorker() {
client := NewHTTPClient(o.address, &HTTPClientConfig{
FollowRedirects: o.config.redirectLimit,
@@ -118,31 +201,22 @@ func (o *HTTPOutput) startWorker() {
ResponseBufferSize: o.config.BufferSize,
})
deathCount := 0
atomic.AddInt64(&o.activeWorkers, 1)
for {
select {
case data := <-o.queue:
o.sendRequest(client, data)
deathCount = 0
case <-time.After(time.Millisecond * 100):
case <-time.After(2 * time.Second):
// When dynamic scaling enabled workers die after 2s of inactivity
if o.config.workers == 0 {
deathCount++
} else {
if o.config.workers > 0 {
continue
}
if deathCount > 20 {
workersCount := atomic.LoadInt64(&o.activeWorkers)
workersCount := atomic.LoadInt64(&o.activeWorkers)
// At least 1 startWorker should be alive
if workersCount != 1 {
atomic.AddInt64(&o.activeWorkers, -1)
return
}
// At least 1 startWorker should be alive
if workersCount != 1 {
atomic.AddInt64(&o.activeWorkers, -1)
return
}
}
}
@@ -162,7 +236,7 @@ func (o *HTTPOutput) Write(data []byte) (n int, err error) {
o.queueStats.Write(len(o.queue))
}
if o.config.workers == 0 {
if !Settings.recognizeTCPSessions && o.config.workers == 0 {
workersCount := atomic.LoadInt64(&o.activeWorkers)
if len(o.queue) > int(workersCount) {
@@ -178,7 +252,7 @@ func (o *HTTPOutput) Read(data []byte) (int, error) {
Debug("[OUTPUT-HTTP] Received response:", string(resp.payload))
header := payloadHeader(ReplayedResponsePayload, resp.uuid, resp.roundTripTime)
header := payloadHeader(ReplayedResponsePayload, resp.uuid, resp.roundTripTime, resp.startedAt)
copy(data[0:len(header)], header)
copy(data[len(header):], resp.payload)
@@ -206,7 +280,7 @@ func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) {
}
if o.config.TrackResponses {
o.responses <- response{resp, uuid, stop.UnixNano() - start.UnixNano()}
o.responses <- response{resp, uuid, start.UnixNano(), stop.UnixNano() - start.UnixNano()}
}
if o.elasticSearch != nil {
output_http_test.go
+52 -2
View File
@@ -57,8 +57,11 @@ func TestHTTPOutput(t *testing.T) {
input.EmitGET()
}
wg.Wait()
if output.(*HTTPOutput).activeWorkers < 50 {
t.Error("Should create workers for each request", output.(*HTTPOutput).activeWorkers)
}
wg.Wait()
close(quit)
Settings.modifierConfig = HTTPModifierConfig{}
@@ -99,7 +102,7 @@ func TestHTTPOutputKeepOriginalHost(t *testing.T) {
Settings.modifierConfig = HTTPModifierConfig{}
}
func TestOutputHTTPSSL(t *testing.T) {
func TestHTTPOutputSSL(t *testing.T) {
wg := new(sync.WaitGroup)
quit := make(chan int)
@@ -125,6 +128,53 @@ func TestOutputHTTPSSL(t *testing.T) {
close(quit)
}
func TestHTTPOutputSessions(t *testing.T) {
wg := new(sync.WaitGroup)
quit := make(chan int)
input := NewTestInput()
input.disableHeaders = true
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
wg.Done()
}))
defer server.Close()
Settings.recognizeTCPSessions = true
output := NewHTTPOutput(server.URL, &HTTPOutputConfig{Debug: true})
Plugins.Inputs = []io.Reader{input}
Plugins.Outputs = []io.Writer{output}
go Start(quit)
uuid1 := []byte("1234567890123456789a0000")
uuid2 := []byte("1234567890123456789d0000")
for i := 0; i < 100; i++ {
wg.Add(1) // OPTIONS should be ignored
copy(uuid1[20:], randByte(4))
input.EmitBytes([]byte("1 " + string(uuid1) + " 1\n" + "GET / HTTP/1.1\r\n\r\n"))
}
for i := 0; i < 100; i++ {
wg.Add(1) // OPTIONS should be ignored
copy(uuid2[20:], randByte(4))
input.EmitBytes([]byte("1 " + string(uuid2) + " 1\n" + "GET / HTTP/1.1\r\n\r\n"))
}
wg.Wait()
if output.(*HTTPOutput).activeWorkers != 2 {
t.Error("Should have only 2 workers", output.(*HTTPOutput).activeWorkers)
}
close(quit)
Settings.recognizeTCPSessions = false
}
func BenchmarkHTTPOutput(b *testing.B) {
wg := new(sync.WaitGroup)
quit := make(chan int)
output_kafka.go
+96
View File
@@ -0,0 +1,96 @@
package main
import (
"encoding/json"
"github.com/Shopify/sarama"
"github.com/buger/gor/proto"
"io"
"log"
"strings"
"time"
)
// KafkaConfig should contains required information to
// build producers.
type KafkaConfig struct {
host string
topic string
}
// KafkaOutput should make producer client.
type KafkaOutput struct {
config *KafkaConfig
producer sarama.AsyncProducer
}
// KafkaMessage should contains catched request information that should be
// passed as Json to Apache Kafka.
type KafkaMessage struct {
ReqURL string `json:"Req_URL"`
ReqMethod string `json:"Req_Method"`
ReqBody string `json:"Req_Body,omitempty"`
ReqHeaders map[string]string `json:"Req_Headers,omitempty"`
}
// KafkaOutputFrequency in milliseconds
const KafkaOutputFrequency = 500
// NewKafkaOutput creates instance of kafka producer client.
func NewKafkaOutput(address string, config *KafkaConfig) io.Writer {
c := sarama.NewConfig()
c.Producer.RequiredAcks = sarama.WaitForLocal
c.Producer.Compression = sarama.CompressionSnappy
c.Producer.Flush.Frequency = KafkaOutputFrequency * time.Millisecond
brokerList := strings.Split(config.host, ",")
producer, err := sarama.NewAsyncProducer(brokerList, c)
if err != nil {
log.Fatalln("Failed to start Sarama(Kafka) producer:", err)
}
o := &KafkaOutput{
config: config,
producer: producer,
}
if Settings.verbose {
// Start infinite loop for tracking errors for kafka producer.
go o.ErrorHandler()
}
return o
}
// ErrorHandler should receive errors
func (o *KafkaOutput) ErrorHandler() {
for err := range o.producer.Errors() {
log.Println("Failed to write access log entry:", err)
}
}
func (o *KafkaOutput) Write(data []byte) (n int, err error) {
headers := make(map[string]string)
proto.ParseHeaders([][]byte{data}, func(header []byte, value []byte) bool {
headers[string(header)] = string(value)
return true
})
req := payloadBody(data)
kafkaMessage := KafkaMessage{
ReqURL: string(proto.Path(req)),
ReqMethod: string(proto.Method(req)),
ReqBody: string(proto.Body(req)),
ReqHeaders: headers,
}
jsonMessage, _ := json.Marshal(&kafkaMessage)
message := sarama.StringEncoder(jsonMessage)
o.producer.Input() <- &sarama.ProducerMessage{
Topic: o.config.topic,
Value: message,
}
return len(message), nil
}
output_null.go
+18
View File
@@ -0,0 +1,18 @@
package main
// NullOutput used for debugging, prints nothing
type NullOutput struct {
}
// NullOutput constructor for NullOutput
func NewNullOutput() (o *NullOutput) {
return new(NullOutput)
}
func (o *NullOutput) Write(data []byte) (int, error) {
return len(data), nil
}
func (o *NullOutput) String() string {
return "Null Output"
}
plugins.go
+16 -2
View File
@@ -95,13 +95,19 @@ func InitPlugins() {
registerPlugin(NewDummyOutput)
}
if Settings.outputNull {
registerPlugin(NewNullOutput)
}
engine := EnginePcap
if Settings.inputRAWEngine == "raw_socket" {
engine = EngineRawSocket
} else if Settings.inputRAWEngine == "pcap_file" {
engine = EnginePcapFile
}
for _, options := range Settings.inputRAW {
registerPlugin(NewRAWInput, options, engine, Settings.inputRAWTrackResponse, time.Duration(0), Settings.inputRAWRealIPHeader)
registerPlugin(NewRAWInput, options, engine, Settings.inputRAWTrackResponse, time.Duration(0), Settings.inputRAWRealIPHeader, Settings.inputRAWProtocol)
}
for _, options := range Settings.inputTCP {
@@ -117,7 +123,7 @@ func InitPlugins() {
}
for _, options := range Settings.outputFile {
registerPlugin(NewFileOutput, options, Settings.outputFileConfig)
registerPlugin(NewFileOutput, options, &Settings.outputFileConfig)
}
for _, options := range Settings.outputS3 {
@@ -141,4 +147,12 @@ func InitPlugins() {
for _, options := range Settings.outputHTTP {
registerPlugin(NewHTTPOutput, options, &Settings.outputHTTPConfig)
}
for _, options := range Settings.outputBinary {
registerPlugin(NewBinaryOutput, options, &Settings.outputBinaryConfig)
}
if Settings.outputKafkaConfig.host != "" && Settings.outputKafkaConfig.topic != "" {
registerPlugin(NewKafkaOutput, "", &Settings.outputKafkaConfig)
}
}
proto/proto.go
+166 -13
View File
@@ -123,7 +123,7 @@ func headerIndex(payload []byte, name []byte) int {
// header return value and positions of header/value start/end.
// If not found, value will be blank, and headerStart will be -1
// Do not support multi-line headers.
func header(payload []byte, name []byte) (value []byte, headerStart, valueStart, headerEnd int) {
func header(payload []byte, name []byte) (value []byte, headerStart, headerEnd, valueStart, valueEnd int) {
headerStart = headerIndex(payload, name)
if headerStart == -1 {
@@ -131,24 +131,164 @@ func header(payload []byte, name []byte) (value []byte, headerStart, valueStart,
}
valueStart = headerStart + len(name) + 1 // Skip ":" after header name
if payload[valueStart] == ' ' { // Ignore empty space after ':'
valueStart++
}
headerEnd = valueStart + bytes.IndexByte(payload[valueStart:], '\n')
if payload[headerEnd-1] == '\r' {
headerEnd -= 1
for valueStart < headerEnd { // Ignore empty space after ':'
if payload[valueStart] == ' ' {
valueStart++
} else {
break
}
}
value = payload[valueStart:headerEnd]
valueEnd = valueStart + bytes.IndexByte(payload[valueStart:], '\n')
if payload[headerEnd-1] == '\r' {
valueEnd--
}
// ignore empty space at end of header value
for valueStart < valueEnd {
if payload[valueEnd-1] == ' ' {
valueEnd--
} else {
break
}
}
value = payload[valueStart:valueEnd]
return
}
// Works only with ASCII
func HeadersEqual(h1 []byte, h2 []byte) bool {
if len(h1) != len(h2) {
return false
}
for i, c1 := range h1 {
c2 := h2[i]
switch int(c1) - int(c2) {
case 0, 32, -32:
default:
return false
}
}
return true
}
// Parsing headers from multiple payloads
func ParseHeaders(payloads [][]byte, cb func(header []byte, value []byte) bool) {
hS := [2]int{0, 0}
hE := [2]int{-1, -1}
vS := [2]int{-1, -1}
vE := [2]int{-1, -1}
i := 0
pIdx := 0
lineBreaks := 0
newLineBreak := true
for {
if len(payloads)-1 < pIdx {
break
}
p := payloads[pIdx]
if len(p)-1 < i {
pIdx++
i = 0
continue
}
switch p[i] {
case '\r', '\n':
newLineBreak = true
lineBreaks++
// End of headers
if lineBreaks == 4 {
return
}
if lineBreaks > 1 {
break
}
vE = [2]int{pIdx, i}
if vS[1] != -1 && vE[1] != -1 &&
hS[1] != -1 && hE[1] != -1 {
var header, value []byte
phS, phE, pvS, pvE := payloads[hS[0]], payloads[hE[0]], payloads[vS[0]], payloads[vE[0]]
// If in same payload
if hS[0] == hE[0] {
header = phS[hS[1]:hE[1]]
} else {
header = make([]byte, len(phS)-hS[1]+hE[1])
copy(header, phS[hS[1]:])
copy(header[len(phS)-hS[1]:], phE[:hE[1]])
}
if vS[0] == vE[0] {
value = pvS[vS[1]:vE[1]]
} else {
value = make([]byte, len(pvS)-vS[1]+vE[1])
copy(value, pvS[vS[1]:])
copy(value[len(pvS)-vS[1]:], pvE[:vE[1]])
}
if !cb(header, value) {
return
}
}
// Header found, reset values
vS = [2]int{-1, -1}
vE = [2]int{-1, -1}
hS = [2]int{-1, -1}
hE = [2]int{-1, -1}
case ':':
if newLineBreak {
hE = [2]int{pIdx, i}
newLineBreak = false
}
default:
lineBreaks = 0
if hS[1] == -1 {
hS = [2]int{pIdx, i}
} else {
if hE[1] == -1 {
break
}
if vS[1] == -1 {
if p[i] == ' ' {
break
}
vS = [2]int{pIdx, i}
}
}
}
i++
}
return
}
// Header returns header value, if header not found, value will be blank
func Header(payload, name []byte) []byte {
val, _, _, _ := header(payload, name)
val, _, _, _, _ := header(payload, name)
return val
}
@@ -156,11 +296,11 @@ func Header(payload, name []byte) []byte {
// SetHeader sets header value. If header not found it creates new one.
// Returns modified request payload
func SetHeader(payload, name, value []byte) []byte {
_, hs, vs, he := header(payload, name)
_, hs, _, vs, ve := header(payload, name)
if hs != -1 {
// If header found we just repace its value
return byteutils.Replace(payload, vs, he, value)
// If header found we just replace its value
return byteutils.Replace(payload, vs, ve, value)
}
return AddHeader(payload, name, value)
@@ -180,6 +320,19 @@ func AddHeader(payload, name, value []byte) []byte {
return byteutils.Insert(payload, mimeStart, header)
}
// DelHeader takes http payload and removes header name from headers section
// Returns modified request payload
func DeleteHeader(payload, name []byte) []byte {
_, hs, he, _, _ := header(payload, name)
if hs != -1 {
newHeader := make([]byte, len(payload)-(he-hs)-1)
copy(newHeader[:hs], payload[:hs])
copy(newHeader[hs:], payload[he+1:])
return newHeader
}
return payload
}
// Body returns request/response body
func Body(payload []byte) []byte {
// 4 -> len(EMPTY_LINE)
@@ -295,7 +448,7 @@ func Status(payload []byte) []byte {
}
var httpMethods []string = []string{
"GET ", "OPTI", "HEAD", "POST", "PUT ", "DELE", "TRAC", "CONN", /* custom methods */"BAN", "PURG",
"GET ", "OPTI", "HEAD", "POST", "PUT ", "DELE", "TRAC", "CONN", "PATC" /* custom methods */, "BAN", "PURG",
}
func IsHTTPPayload(payload []byte) bool {
proto/proto_test.go
+120 -1
View File
@@ -2,6 +2,7 @@ package proto
import (
"bytes"
"reflect"
"testing"
)
@@ -16,6 +17,13 @@ func TestHeader(t *testing.T) {
t.Error("Should find header value")
}
// Value with space at end
payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7 \r\nHost: www.w3.org\r\n\r\na=1&b=2")
if val = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("7")) {
t.Error("Should find header value without space after 7")
}
// Value without space at start
payload = []byte("POST /post HTTP/1.1\r\nContent-Length:7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
@@ -38,7 +46,7 @@ func TestHeader(t *testing.T) {
}
// Header not found
if _, headerStart, _, _ = header(payload, []byte("Not-Found")); headerStart != -1 {
if _, headerStart, _, _, _ = header(payload, []byte("Not-Found")); headerStart != -1 {
t.Error("Should not found header")
}
@@ -97,6 +105,117 @@ func TestSetHeader(t *testing.T) {
}
}
func TestDeleteHeader(t *testing.T) {
var payload, payloadAfter []byte
payload = []byte("POST /post HTTP/1.1\r\nUser-Agent: Gor\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
payloadAfter = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
if payload = DeleteHeader(payload, []byte("User-Agent")); !bytes.Equal(payload, payloadAfter) {
t.Error("Should delete header if found", string(payload), string(payloadAfter))
}
//Whitespace at end of User-Agent
payload = []byte("POST /post HTTP/1.1\r\nUser-Agent: Gor \r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
payloadAfter = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
if payload = DeleteHeader(payload, []byte("User-Agent")); !bytes.Equal(payload, payloadAfter) {
t.Error("Should delete header if found", string(payload), string(payloadAfter))
}
}
func TestParseHeaders(t *testing.T) {
payload := [][]byte{[]byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.or"), []byte("g\r\nUser-Ag"), []byte("ent:Chrome\r\n\r\n"), []byte("Fake-Header: asda")}
headers := make(map[string]string)
ParseHeaders(payload, func(header []byte, value []byte) bool {
headers[string(header)] = string(value)
return true
})
expected := map[string]string{
"Content-Length": "7",
"Host": "www.w3.org",
"User-Agent": "Chrome",
}
if !reflect.DeepEqual(headers, expected) {
t.Error("Headers do not properly parsed", headers)
}
}
func TestParseHeadersWithComplexUserAgent(t *testing.T) {
// User-Agent could contain inside ':'
// Parser should wait for \r\n
payload := [][]byte{[]byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.or"), []byte("g\r\nUser-Ag"), []byte("ent:Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko\r\n\r\n"), []byte("Fake-Header: asda")}
headers := make(map[string]string)
ParseHeaders(payload, func(header []byte, value []byte) bool {
headers[string(header)] = string(value)
return true
})
expected := map[string]string{
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko",
}
if expected["User-Agent"] != headers["User-Agent"] {
t.Errorf("Header 'User-Agent' expected '%s' and parsed: '%s'", expected["User-Agent"], headers["User-Agent"])
}
}
func TestParseHeadersWithOrigin(t *testing.T) {
// User-Agent could contain inside ':'
// Parser should wait for \r\n
payload := [][]byte{[]byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.or"), []byte("g\r\nReferrer: http://127.0.0.1:3000\r\nOrigi"), []byte("n: https://www.example.com\r\nUser-Ag"), []byte("ent:Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko\r\n\r\n"), []byte("in:https://www.example.com\r\n\r\n"), []byte("Fake-Header: asda")}
headers := make(map[string]string)
ParseHeaders(payload, func(header []byte, value []byte) bool {
headers[string(header)] = string(value)
return true
})
expected := map[string]string{
"Origin": "https://www.example.com",
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko",
"Referrer": "http://127.0.0.1:3000",
}
if expected["Referrer"] != headers["Referrer"] {
t.Errorf("Header 'Referrer' expected '%s' and parsed: '%s'", expected["Referrer"], headers["Referrer"])
}
if expected["Origin"] != headers["Origin"] {
t.Errorf("Header 'Origin' expected '%s' and parsed: '%s'", expected["Origin"], headers["Origin"])
}
if expected["User-Agent"] != headers["User-Agent"] {
t.Errorf("Header 'User-Agent' expected '%s' and parsed: '%s'", expected["User-Agent"], headers["User-Agent"])
}
}
func TestHeaderEquals(t *testing.T) {
tests := []struct {
h1 string
h2 string
equals bool
}{
{"Content-Length", "content-length", true},
{"content-length", "Content-Length", true},
{"content-Pength", "Content-Length", false},
{"Host", "Content-Length", false},
}
for _, tc := range tests {
if HeadersEqual([]byte(tc.h1), []byte(tc.h2)) != tc.equals {
t.Error(tc)
}
}
}
func TestPath(t *testing.T) {
var path, payload []byte
protocol.go
+38 -8
View File
@@ -13,14 +13,18 @@ const (
ReplayedResponsePayload = '3'
)
func uuid() []byte {
b := make([]byte, 20)
func randByte(len int) []byte {
b := make([]byte, len / 2)
rand.Read(b)
uuid := make([]byte, 40)
hex.Encode(uuid, b)
h := make([]byte, len)
hex.Encode(h, b)
return uuid
return h
}
func uuid() []byte {
return randByte(24)
}
var payloadSeparator = "\n🐵🙈🙉\n"
@@ -42,13 +46,24 @@ func payloadScanner(data []byte, atEOF bool) (advance int, token []byte, err err
}
// Timing is request start or round-trip time, depending on payloadType
func payloadHeader(payloadType byte, uuid []byte, timing int64) (header []byte) {
sTime := strconv.FormatInt(timing, 10)
func payloadHeader(payloadType byte, uuid []byte, timing int64, latency int64) (header []byte) {
var sTime, sLatency string
sTime = strconv.FormatInt(timing, 10)
if latency != -1 {
sLatency = strconv.FormatInt(latency, 10)
}
//Example:
// 3 f45590522cd1838b4a0d5c5aab80b77929dea3b3 1231\n
// `+ 1` indicates space characters or end of line
header = make([]byte, 1+1+len(uuid)+1+len(sTime)+1)
headerLen := 1 + 1 + len(uuid) + 1 + len(sTime) + 1
if latency != -1 {
headerLen += len(sLatency) + 1
}
header = make([]byte, headerLen)
header[0] = payloadType
header[1] = ' '
header[2+len(uuid)] = ' '
@@ -57,6 +72,11 @@ func payloadHeader(payloadType byte, uuid []byte, timing int64) (header []byte)
copy(header[2:], uuid)
copy(header[3+len(uuid):], sTime)
if latency != -1 {
header[3+len(uuid)+len(sTime)] = ' '
copy(header[4+len(uuid)+len(sTime):], sLatency)
}
return header
}
@@ -73,6 +93,16 @@ func payloadMeta(payload []byte) [][]byte {
return bytes.Split(payload[:headerSize], []byte{' '})
}
func payloadID(payload []byte) []byte {
idx := bytes.IndexByte(payload[2:], ' ')
if idx == -1 {
return []byte{}
}
return payload[2: 2 + idx]
}
func isOriginPayload(payload []byte) bool {
switch payload[0] {
case RequestPayload, ResponsePayload:
raw_socket_listener/listener.go
+282 -99
View File
@@ -16,6 +16,7 @@ import (
"bytes"
"encoding/binary"
"fmt"
"github.com/buger/gor-pro/proto"
"github.com/google/gopacket"
"github.com/google/gopacket/layers"
"github.com/google/gopacket/pcap"
@@ -32,6 +33,12 @@ import (
var _ = fmt.Println
type packet struct {
srcIP []byte
data []byte
timestamp time.Time
}
// Listener handle traffic capture
type Listener struct {
mu sync.Mutex
@@ -52,7 +59,7 @@ type Listener struct {
respWithoutReq map[uint32]tcpID
// Messages ready to be send to client
packetsChan chan []byte
packetsChan chan *packet
// Messages ready to be send to client
messagesChan chan *TCPMessage
@@ -68,6 +75,8 @@ type Listener struct {
quit chan bool
readyCh chan bool
protocol TCPProtocol
}
type request struct {
@@ -80,13 +89,14 @@ type request struct {
const (
EngineRawSocket = 1 << iota
EnginePcap
EnginePcapFile
)
// NewListener creates and initializes new Listener object
func NewListener(addr string, port string, engine int, trackResponse bool, expire time.Duration) (l *Listener) {
func NewListener(addr string, port string, engine int, trackResponse bool, expire time.Duration, protocol TCPProtocol) (l *Listener) {
l = &Listener{}
l.packetsChan = make(chan []byte, 10000)
l.packetsChan = make(chan *packet, 10000)
l.messagesChan = make(chan *TCPMessage, 10000)
l.quit = make(chan bool)
l.readyCh = make(chan bool, 1)
@@ -97,6 +107,7 @@ func NewListener(addr string, port string, engine int, trackResponse bool, expir
l.respAliases = make(map[uint32]*TCPMessage)
l.respWithoutReq = make(map[uint32]tcpID)
l.trackResponse = trackResponse
l.protocol = protocol
l.addr = addr
_port, _ := strconv.Atoi(port)
@@ -113,10 +124,10 @@ func NewListener(addr string, port string, engine int, trackResponse bool, expir
// Special case for testing
if l.port != 0 {
switch engine {
case EngineRawSocket:
go l.readRAWSocket()
case EnginePcap:
go l.readPcap()
case EnginePcapFile:
go l.readPcapFile()
default:
log.Fatal("Unknown traffic interception engine:", engine)
}
@@ -135,9 +146,9 @@ func (t *Listener) listen() {
t.conn.Close()
}
return
case data := <-t.packetsChan:
packet := ParseTCPPacket(data[:16], data[16:])
t.processTCPPacket(packet)
case packet := <-t.packetsChan:
tcpPacket := ParseTCPPacket(packet.srcIP, packet.data, packet.timestamp)
t.processTCPPacket(tcpPacket)
case <-gcTicker:
now := time.Now()
@@ -172,7 +183,14 @@ func (t *Listener) dispatchMessage(message *TCPMessage) {
t.deleteMessage(message)
// log.Println("Dispatching, message", message.Start.UnixNano(), message.Seq, message.Ack, string(message.Bytes()))
if t.protocol == ProtocolHTTP && !message.complete {
if !message.IsIncoming {
delete(t.respAliases, message.Ack)
delete(t.respWithoutReq, message.Ack)
}
return
}
if message.IsIncoming {
// If there were response before request
@@ -182,10 +200,10 @@ func (t *Listener) dispatchMessage(message *TCPMessage) {
if resp, rok := t.messages[respID]; rok {
// if resp.AssocMessage == nil {
// log.Println("FOUND RESPONSE")
resp.AssocMessage = message
message.AssocMessage = resp
resp.setAssocMessage(message)
message.setAssocMessage(resp)
if resp.IsFinished() {
if resp.complete {
defer t.dispatchMessage(resp)
}
// }
@@ -193,14 +211,14 @@ func (t *Listener) dispatchMessage(message *TCPMessage) {
}
if resp, ok := t.messages[message.ResponseID]; ok {
resp.AssocMessage = message
resp.setAssocMessage(message)
}
}
} else {
if message.AssocMessage == nil {
if responseRequest, ok := t.respAliases[message.Ack]; ok {
message.AssocMessage = responseRequest
responseRequest.AssocMessage = message
message.setAssocMessage(responseRequest)
responseRequest.setAssocMessage(message)
}
}
@@ -244,6 +262,28 @@ func (e *DeviceNotFoundError) Error() string {
return msg
}
func isLoopback(device pcap.Interface) bool {
if len(device.Addresses) == 0 {
return false
}
switch device.Addresses[0].IP.String() {
case "127.0.0.1", "::1":
return true
}
return false
}
func listenAllInterfaces(addr string) bool {
switch addr {
case "", "0.0.0.0", "[::]", "::":
return true
default:
return false
}
}
func findPcapDevices(addr string) (interfaces []pcap.Interface, err error) {
devices, err := pcap.FindAllDevs()
if err != nil {
@@ -251,7 +291,7 @@ func findPcapDevices(addr string) (interfaces []pcap.Interface, err error) {
}
for _, device := range devices {
if (addr == "" || addr == "0.0.0.0" || addr == "[::]" || addr == "::") && len(device.Addresses) > 0 {
if listenAllInterfaces(addr) && len(device.Addresses) > 0 || isLoopback(device) {
interfaces = append(interfaces, device)
continue
}
@@ -299,12 +339,26 @@ func (t *Listener) readPcap() {
t.pcapHandles = append(t.pcapHandles, handle)
var bpfDstHost, bpfSrcHost string
for i, addr := range device.Addresses {
bpfDstHost += "dst host " + addr.IP.String()
bpfSrcHost += "src host " + addr.IP.String()
if i != len(device.Addresses) - 1 {
bpfDstHost += " or "
bpfSrcHost += " or "
var loopback = isLoopback(device)
if loopback {
var allAddr []string
for _, dc := range devices {
for _, addr := range dc.Addresses {
allAddr = append(allAddr, "(dst host "+addr.IP.String()+" and src host "+addr.IP.String()+")")
}
}
bpfDstHost = strings.Join(allAddr, " or ")
bpfSrcHost = bpfDstHost
} else {
for i, addr := range device.Addresses {
bpfDstHost += "dst host " + addr.IP.String()
bpfSrcHost += "src host " + addr.IP.String()
if i != len(device.Addresses)-1 {
bpfDstHost += " or "
bpfSrcHost += " or "
}
}
}
@@ -325,8 +379,16 @@ func (t *Listener) readPcap() {
}
t.mu.Unlock()
linkType := handle.LinkType()
source := gopacket.NewPacketSource(handle, linkType)
var decoder gopacket.Decoder
// Special case for tunnel interface https://github.com/google/gopacket/issues/99
if handle.LinkType() == 12 {
decoder = layers.LayerTypeIPv4
} else {
decoder = handle.LinkType()
}
source := gopacket.NewPacketSource(handle, decoder)
source.Lazy = true
source.NoCopy = true
@@ -343,14 +405,32 @@ func (t *Listener) readPcap() {
continue
}
if linkType == layers.LinkTypeEthernet {
// Skip ethernet layer, 14 bytes
data = packet.Data()[14:]
} else if linkType == layers.LinkTypeNull || linkType == layers.LinkTypeLoop {
data = packet.Data()[4:]
// We should remove network layer before parsing TCP/IP data
var of int
switch decoder {
case layers.LinkTypeEthernet:
of = 14
case layers.LinkTypePPP:
of = 1
case layers.LinkTypeFDDI:
of = 13
case layers.LinkTypeNull:
of = 4
case layers.LinkTypeLoop:
of = 4
case layers.LinkTypeRaw:
of = 0
case layers.LinkTypeLinuxSLL:
of = 16
default:
log.Println("Unknown packet layer", packet)
break
}
data = packet.Data()[of:]
version := uint8(data[0]) >> 4
ipLength := int(binary.BigEndian.Uint16(data[2:4]))
if version == 4 {
ihl := uint8(data[0]) & 0x0F
@@ -362,6 +442,24 @@ func (t *Listener) readPcap() {
srcIP = data[12:16]
dstIP = data[16:20]
// Too small IP packet
if ipLength < 20 {
continue
}
// Invalid length
if int(ihl*4) > ipLength {
continue
}
if cmp := len(data) - ipLength; cmp > 0 {
data = data[:ipLength]
} else if cmp < 0 {
// Truncated packet
continue
}
data = data[ihl*4:]
} else {
// Truncated IP info
@@ -376,15 +474,16 @@ func (t *Listener) readPcap() {
}
// Truncated TCP info
if len(data) < 13 {
if len(data) <= 13 {
continue
}
dataOffset := (data[12] & 0xF0) >> 4
isFIN := data[13]&0x01 != 0
// We need only packets with data inside
// Check that the buffer is larger than the size of the TCP header
if len(data) > int(dataOffset*4) {
if len(data) > int(dataOffset*4) || isFIN {
if !bpfSupported {
destPort := binary.BigEndian.Uint16(data[2:4])
srcPort := binary.BigEndian.Uint16(data[0:2])
@@ -404,10 +503,26 @@ func (t *Listener) readPcap() {
}
addrMatched := false
for _, a := range device.Addresses {
if a.IP.Equal(net.IP(addrCheck)) {
addrMatched = true
break
if loopback {
for _, dc := range devices {
if addrMatched {
break
}
for _, a := range dc.Addresses {
if a.IP.Equal(net.IP(addrCheck)) {
addrMatched = true
break
}
}
}
addrMatched = true
} else {
for _, a := range device.Addresses {
if a.IP.Equal(net.IP(addrCheck)) {
addrMatched = true
break
}
}
}
@@ -416,11 +531,7 @@ func (t *Listener) readPcap() {
}
}
newBuf := make([]byte, len(data)+16)
copy(newBuf[:16], srcIP)
copy(newBuf[16:], data)
t.packetsChan <- newBuf
t.packetsChan <- t.buildPacket(srcIP, data, packet.Metadata().Timestamp)
}
}
}(d)
@@ -430,6 +541,64 @@ func (t *Listener) readPcap() {
t.readyCh <- true
}
func (t *Listener) readPcapFile() {
if handle, err := pcap.OpenOffline(t.addr); err != nil {
log.Fatal(err)
} else {
t.readyCh <- true
packetSource := gopacket.NewPacketSource(handle, handle.LinkType())
for {
packet, err := packetSource.NextPacket()
if err == io.EOF {
break
} else if err != nil {
log.Println("Error:", err)
continue
}
var addr, data []byte
if tcpLayer := packet.Layer(layers.LayerTypeTCP); tcpLayer != nil {
tcp, _ := tcpLayer.(*layers.TCP)
data = append(tcp.LayerContents(), tcp.LayerPayload()...)
if tcp.SrcPort >= 32768 && tcp.SrcPort <= 61000 {
copy(data[0:2], []byte{0, 0})
copy(data[2:4], []byte{0, 1})
} else {
copy(data[0:2], []byte{0, 1})
copy(data[2:4], []byte{0, 0})
}
} else {
continue
}
if ipLayer := packet.Layer(layers.LayerTypeIPv4); ipLayer != nil {
ip, _ := ipLayer.(*layers.IPv4)
addr = ip.SrcIP
} else if ipLayer = packet.Layer(layers.LayerTypeIPv6); ipLayer != nil {
ip, _ := ipLayer.(*layers.IPv6)
addr = ip.SrcIP
} else {
// log.Println("Can't find IP layer", packet)
continue
}
dataOffset := (data[12] & 0xF0) >> 4
isFIN := data[13]&0x01 != 0
// We need only packets with data inside
// Check that the buffer is larger than the size of the TCP header
if len(data) <= int(dataOffset*4) && !isFIN {
continue
}
t.packetsChan <- t.buildPacket(addr, data, packet.Metadata().Timestamp)
}
}
}
func (t *Listener) readRAWSocket() {
conn, e := net.ListenPacket("ip:tcp", t.addr)
t.conn = conn
@@ -458,16 +627,26 @@ func (t *Listener) readRAWSocket() {
if n > 0 {
if t.isValidPacket(buf[:n]) {
newBuf := make([]byte, n+16)
copy(newBuf[16:], buf[:n])
copy(newBuf[:16], []byte(addr.(*net.IPAddr).IP))
t.packetsChan <- newBuf
t.packetsChan <- t.buildPacket([]byte(addr.(*net.IPAddr).IP), buf[:n], time.Now())
}
}
}
}
func (t *Listener) buildPacket(packetSrcIP []byte, packetData []byte, timestamp time.Time) *packet {
copyPacketSrcIP := make([]byte, 16)
copyPacketData := make([]byte, len(packetData))
copy(copyPacketSrcIP, packetSrcIP)
copy(copyPacketData, packetSrcIP)
return &packet{
srcIP: packetSrcIP,
data: packetData,
timestamp:timestamp,
}
}
func (t *Listener) isValidPacket(buf []byte) bool {
// To avoid full packet parsing every time, we manually parsing values needed for packet filtering
// http://en.wikipedia.org/wiki/Transmission_Control_Protocol
@@ -490,9 +669,6 @@ func (t *Listener) isValidPacket(buf []byte) bool {
return false
}
var bExpect100ContinueCheck = []byte("Expect: 100-continue")
var bPOST = []byte("POST")
// Trying to add packet to existing message or creating new message
//
// For TCP message unique id is Acknowledgment number (see tcp_packet.go)
@@ -504,34 +680,42 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) {
}
}()
// log.Println("Processing packet:", packet.Ack, packet.Seq, packet.ID)
var message *TCPMessage
isIncoming := packet.DestPort == t.port
// Seek for 100-expect chunks
if parentAck, ok := t.seqWithData[packet.Seq]; ok {
// In case if non-first data chunks comes first
for _, m := range t.messages {
if m.Ack == packet.Ack && bytes.Equal(m.packets[0].Addr, packet.Addr) {
t.deleteMessage(m)
if t.protocol == ProtocolHTTP {
// Seek for 100-expect chunks
if parentAck, ok := t.seqWithData[packet.Seq]; ok {
// In case if non-first data chunks comes first
for _, m := range t.messages {
if m.Ack == packet.Ack && bytes.Equal(m.packets[0].Addr, packet.Addr) {
t.deleteMessage(m)
if m.AssocMessage != nil {
m.AssocMessage.AssocMessage = nil
}
if m.AssocMessage != nil {
m.setAssocMessage(nil)
}
for _, pkt := range m.packets {
// log.Println("Updating ack", parentAck, pkt.Ack)
pkt.UpdateAck(parentAck)
// Re-queue this packets
t.processTCPPacket(pkt)
for _, pkt := range m.packets {
// log.Println("Updating ack", parentAck, pkt.Ack)
pkt.UpdateAck(parentAck)
// Re-queue this packets
t.processTCPPacket(pkt)
}
}
}
}
t.ackAliases[packet.Ack] = parentAck
packet.UpdateAck(parentAck)
t.ackAliases[packet.Ack] = parentAck
packet.UpdateAck(parentAck)
}
}
if isIncoming && packet.IsFIN {
if ma, ok := t.respAliases[packet.Seq]; ok {
if ma.packets[0].SrcPort == packet.SrcPort {
packet.UpdateAck(ma.Ack)
}
}
}
if alias, ok := t.ackAliases[packet.Ack]; ok {
@@ -547,13 +731,13 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) {
message, ok := t.messages[packet.ID]
if !ok {
message = NewTCPMessage(packet.Seq, packet.Ack, isIncoming)
message = NewTCPMessage(packet.Seq, packet.Ack, isIncoming, t.protocol, packet.timestamp)
t.messages[packet.ID] = message
if !isIncoming {
if responseRequest != nil {
message.AssocMessage = responseRequest
responseRequest.AssocMessage = message
message.setAssocMessage(responseRequest)
responseRequest.setAssocMessage(message)
} else {
t.respWithoutReq[packet.Ack] = packet.ID
}
@@ -564,35 +748,31 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) {
message.AddPacket(packet)
// Handling Expect: 100-continue requests
if len(packet.Data) > 4 && bytes.Equal(packet.Data[0:4], bPOST) {
// reading last 20 bytes (not counting CRLF): last header value (if no body presented)
if bytes.Equal(packet.Data[len(packet.Data)-24:len(packet.Data)-4], bExpect100ContinueCheck) {
seq := packet.Seq + uint32(len(packet.Data))
t.seqWithData[seq] = packet.Ack
message.DataSeq = seq
if t.protocol == ProtocolHTTP && message.expectType == httpExpect100Continue && len(message.packets) == message.headerPacket+1 {
seq := packet.Seq + uint32(message.Size())
t.seqWithData[seq] = packet.Ack
message.DataSeq = seq
message.complete = false
// In case if sequence packet came first
for _, m := range t.messages {
if m.Seq == seq {
t.deleteMessage(m)
if m.AssocMessage != nil {
message.AssocMessage = m.AssocMessage
}
// log.Println("2: Adding ack alias:", m.Ack, packet.Ack)
t.ackAliases[m.Ack] = packet.Ack
// In case if sequence packet came first
for _, m := range t.messages {
if m.Seq == seq {
t.deleteMessage(m)
if m.AssocMessage != nil {
message.setAssocMessage(m.AssocMessage)
}
// log.Println("2: Adding ack alias:", m.Ack, packet.Ack)
t.ackAliases[m.Ack] = packet.Ack
for _, pkt := range m.packets {
pkt.UpdateAck(packet.Ack)
message.AddPacket(pkt)
}
for _, pkt := range m.packets {
pkt.UpdateAck(packet.Ack)
message.AddPacket(pkt)
}
}
// Removing `Expect: 100-continue` header
packet.Data = append(packet.Data[:len(packet.Data)-24], packet.Data[len(packet.Data)-2:]...)
// log.Println(string(packet.Data))
}
// Removing `Expect: 100-continue` header
packet.Data = proto.DeleteHeader(packet.Data, bExpectHeader)
}
// log.Println("Received message:", string(message.Bytes()), message.ID(), t.messages)
@@ -608,15 +788,18 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) {
}
// If message contains only single packet immediately dispatch it
if message.IsFinished() {
if message.complete {
// log.Println("COMPLETE!", isIncoming, message)
if isIncoming {
// log.Println("I'm finished", string(message.Bytes()), message.ResponseID, t.messages)
if t.trackResponse {
// log.Println("Found response!", message.ResponseID, t.messages)
if resp, ok := t.messages[message.ResponseID]; ok {
t.dispatchMessage(message)
if resp.IsFinished() {
if resp.complete {
t.dispatchMessage(resp)
}
t.dispatchMessage(message)
}
} else {
t.dispatchMessage(message)
@@ -627,7 +810,7 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) {
}
if req, ok := t.messages[message.AssocMessage.ID()]; ok {
if req.IsFinished() {
if req.complete {
t.dispatchMessage(req)
t.dispatchMessage(message)
}
raw_socket_listener/listener_test.go
+119 -95
View File
@@ -12,16 +12,56 @@ import (
func TestRawListenerInput(t *testing.T) {
var req, resp *TCPMessage
listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP)
defer listener.Close()
reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now())
respAck := reqPacket.Seq + uint32(len(reqPacket.Data))
respPacket := buildPacket(false, respAck, reqPacket.Seq+1, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now())
listener.packetsChan <- reqPacket.dump()
listener.packetsChan <- respPacket.dump()
select {
case req = <-listener.messagesChan:
case <-time.After(time.Millisecond):
t.Error("Should return request immediately")
return
}
if !req.IsIncoming {
t.Error("Should be request")
}
select {
case resp = <-listener.messagesChan:
case <-time.After(20 * time.Millisecond):
t.Error("Should return response immediately")
return
}
if resp.IsIncoming {
t.Error("Should be response")
}
}
func TestRawListenerInputResponseByClose(t *testing.T) {
var req, resp *TCPMessage
listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond)
defer listener.Close()
reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"))
reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now())
respAck := reqPacket.Seq + uint32(len(reqPacket.Data))
respPacket := buildPacket(false, respAck, reqPacket.Seq+1, []byte("HTTP/1.1 200 OK\r\n\r\n"))
respPacket := buildPacket(false, respAck, reqPacket.Seq+1, []byte("HTTP/1.1 200 OK\r\nConnection: close\r\n\r\nasd"), time.Now())
finPacket := buildPacket(false, respAck, reqPacket.Seq+2, []byte(""), time.Now())
finPacket.IsFIN = true
listener.packetsChan <- reqPacket.Dump()
listener.packetsChan <- respPacket.Dump()
listener.packetsChan <- reqPacket.dump()
listener.packetsChan <- respPacket.dump()
listener.packetsChan <- finPacket.dump()
select {
case req = <-listener.messagesChan:
@@ -49,12 +89,12 @@ func TestRawListenerInput(t *testing.T) {
func TestRawListenerInputWithoutResponse(t *testing.T) {
var req *TCPMessage
listener := NewListener("", "0", EnginePcap, false, 10*time.Millisecond)
listener := NewListener("", "0", EnginePcap, false, 10*time.Millisecond, ProtocolHTTP)
defer listener.Close()
reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"))
reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now())
listener.packetsChan <- reqPacket.Dump()
listener.packetsChan <- reqPacket.dump()
select {
case req = <-listener.messagesChan:
@@ -71,15 +111,15 @@ func TestRawListenerInputWithoutResponse(t *testing.T) {
func TestRawListenerResponse(t *testing.T) {
var req, resp *TCPMessage
listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond)
listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP)
defer listener.Close()
reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"))
respPacket := buildPacket(false, 1+uint32(len(reqPacket.Data)), 2, []byte("HTTP/1.1 200 OK\r\n\r\n"))
reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now())
respPacket := buildPacket(false, 1+uint32(len(reqPacket.Data)), 2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now())
// If response packet comes before request
listener.packetsChan <- respPacket.Dump()
listener.packetsChan <- reqPacket.Dump()
listener.packetsChan <- respPacket.dump()
listener.packetsChan <- reqPacket.dump()
select {
case req = <-listener.messagesChan:
@@ -108,85 +148,69 @@ func TestRawListenerResponse(t *testing.T) {
}
}
func TestRawListener100Continue(t *testing.T) {
var req, resp *TCPMessage
listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond)
func TestShort100Continue(t *testing.T) {
listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP)
defer listener.Close()
reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\nExpect: 100-continue\r\n\r\n"))
reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\nExpect: 100-continue\r\n\r\n"), time.Now())
// Packet with data have different Seq
reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("a"))
reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+1, []byte("b"))
reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("a"), time.Now())
reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+1, []byte("b"), time.Now())
respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n"))
respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n\r\n"), time.Now())
// panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data))))
respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n"))
respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now())
listener.packetsChan <- reqPacket1.Dump()
listener.packetsChan <- reqPacket2.Dump()
listener.packetsChan <- reqPacket3.Dump()
result := []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab")
listener.packetsChan <- respPacket1.Dump()
listener.packetsChan <- respPacket2.Dump()
select {
case req = <-listener.messagesChan:
break
case <-time.After(11 * time.Millisecond):
t.Error("Should return request after expire time")
return
}
if !bytes.Equal(req.Bytes(), []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab")) {
t.Error("Should receive full message", string(req.Bytes()))
}
if !req.IsIncoming {
t.Error("Should be request")
}
select {
case resp = <-listener.messagesChan:
break
case <-time.After(21 * time.Millisecond):
t.Error("Should return response after expire time")
return
}
if resp.IsIncoming {
t.Error("Should be response")
}
if !bytes.Equal(resp.UUID(), req.UUID()) {
t.Error("Resp and Req UUID should be equal")
}
testRawListener100Continue(t, listener, result, reqPacket1, reqPacket2, reqPacket3, respPacket1, respPacket2)
}
// Response comes before Request
func TestRawListener100ContinueWrongOrder(t *testing.T) {
var req, resp *TCPMessage
listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond)
func Test100ContinueWrongOrder(t *testing.T) {
listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP)
defer listener.Close()
reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\nExpect: 100-continue\r\n\r\n"))
reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\nExpect: 100-continue\r\n\r\n"), time.Now())
// Packet with data have different Seq
reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("a"))
reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+1, []byte("b"))
reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("a"), time.Now())
reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+1, []byte("b"), time.Now())
respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n"))
respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n"), time.Now())
// panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data))))
respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n"))
respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now())
listener.packetsChan <- respPacket1.Dump()
listener.packetsChan <- respPacket2.Dump()
result := []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab")
listener.packetsChan <- reqPacket1.Dump()
listener.packetsChan <- reqPacket2.Dump()
listener.packetsChan <- reqPacket3.Dump()
testRawListener100Continue(t, listener, result, respPacket1, respPacket2, reqPacket1, reqPacket2, reqPacket3)
}
func TestAlt100ContinueHeaderOrder(t *testing.T) {
listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP)
defer listener.Close()
reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 2\r\n\r\n"), time.Now())
// Packet with data have different Seq
reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("a"), time.Now())
reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+1, []byte("b"), time.Now())
respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n"), time.Now())
// panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data))))
respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now())
result := []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab")
testRawListener100Continue(t, listener, result, reqPacket1, reqPacket2, reqPacket3, respPacket1, respPacket2)
}
func testRawListener100Continue(t *testing.T, listener *Listener, result []byte, packets ...*TCPPacket) {
var req, resp *TCPMessage
for _, p := range packets {
listener.packetsChan <- p.dump()
}
select {
case req = <-listener.messagesChan:
@@ -196,7 +220,7 @@ func TestRawListener100ContinueWrongOrder(t *testing.T) {
return
}
if !bytes.Equal(req.Bytes(), []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab")) {
if !bytes.Equal(req.Bytes(), result) {
t.Error("Should receive full message", string(req.Bytes()))
}
@@ -225,7 +249,7 @@ func testChunkedSequence(t *testing.T, listener *Listener, packets ...*TCPPacket
var r, req, resp *TCPMessage
for _, p := range packets {
listener.packetsChan <- p.Dump()
listener.packetsChan <- p.dump()
}
select {
@@ -325,26 +349,26 @@ func permutation(n int, list []*TCPPacket) []*TCPPacket {
// Response comes before Request
func TestRawListenerChunkedWrongOrder(t *testing.T) {
listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond)
listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP)
defer listener.Close()
reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\nExpect: 100-continue\r\n\r\n"))
reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\nExpect: 100-continue\r\n\r\n"), time.Now())
// Packet with data have different Seq
reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("1\r\na\r\n"))
reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+uint32(len(reqPacket2.Data)), []byte("1\r\nb\r\n"))
reqPacket4 := buildPacket(true, 2, reqPacket3.Seq+uint32(len(reqPacket3.Data)), []byte("0\r\n\r\n"))
reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("1\r\na\r\n"), time.Now())
reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+uint32(len(reqPacket2.Data)), []byte("1\r\nb\r\n"), time.Now())
reqPacket4 := buildPacket(true, 2, reqPacket3.Seq+uint32(len(reqPacket3.Data)), []byte("0\r\n\r\n"), time.Now())
respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n"))
respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n\r\n"), time.Now())
// panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data))))
respPacket2 := buildPacket(false, reqPacket4.Seq+5 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n"))
respPacket2 := buildPacket(false, reqPacket4.Seq+5 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now())
// Should re-construct message from all possible combinations
for i := 0; i < 6*5*4*3*2*1; i++ {
if i < 54 || i > 57 {
continue
}
// if i < 54 || i > 57 {
// continue
// }
packets := permutation(i, []*TCPPacket{reqPacket1, reqPacket2, reqPacket3, reqPacket4, respPacket1, respPacket2})
@@ -357,13 +381,13 @@ func chunkedPostMessage() []*TCPPacket {
ack := uint32(rand.Int63())
seq := uint32(rand.Int63())
reqPacket1 := buildPacket(true, ack, seq, []byte("POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"))
reqPacket1 := buildPacket(true, ack, seq, []byte("POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"), time.Now())
// Packet with data have different Seq
reqPacket2 := buildPacket(true, ack, seq+47, []byte("1\r\na\r\n"))
reqPacket3 := buildPacket(true, ack, reqPacket2.Seq+5, []byte("1\r\nb\r\n"))
reqPacket4 := buildPacket(true, ack, reqPacket3.Seq+5, []byte("0\r\n\r\n"))
reqPacket2 := buildPacket(true, ack, seq+47, []byte("1\r\na\r\n"), time.Now())
reqPacket3 := buildPacket(true, ack, reqPacket2.Seq+5, []byte("1\r\nb\r\n"), time.Now())
reqPacket4 := buildPacket(true, ack, reqPacket3.Seq+5, []byte("0\r\n\r\n"), time.Now())
respPacket := buildPacket(false, reqPacket4.Seq+5 /* len of data */, ack, []byte("HTTP/1.1 200 OK\r\n"))
respPacket := buildPacket(false, reqPacket4.Seq+5 /* len of data */, ack, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now())
return []*TCPPacket{
reqPacket1, reqPacket2, reqPacket3, reqPacket4, respPacket,
@@ -385,8 +409,8 @@ func postMessage() []*TCPPacket {
}
return []*TCPPacket{
buildPacket(true, ack, seq, data),
buildPacket(false, seq+uint32(len(data)), seq2, []byte("HTTP/1.1 200 OK\r\n")),
buildPacket(true, ack, seq, data, time.Now()),
buildPacket(false, seq+uint32(len(data)), seq2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()),
}
}
@@ -396,14 +420,14 @@ func getMessage() []*TCPPacket {
seq := uint32(rand.Int63())
return []*TCPPacket{
buildPacket(true, ack, seq, []byte("GET / HTTP/1.1\r\n\r\n")),
buildPacket(false, seq+18, seq2, []byte("HTTP/1.1 200 OK\r\n")),
buildPacket(true, ack, seq, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now()),
buildPacket(false, seq+18, seq2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()),
}
}
// Response comes before Request
func TestRawListenerBench(t *testing.T) {
l := NewListener("", "0", EnginePcap, true, 200*time.Millisecond)
l := NewListener("", "0", EnginePcap, true, 200*time.Millisecond, ProtocolHTTP)
defer l.Close()
// Should re-construct message from all possible combinations
@@ -428,7 +452,7 @@ func TestRawListenerBench(t *testing.T) {
}
}
l.packetsChan <- p.Dump()
l.packetsChan <- p.dump()
time.Sleep(time.Millisecond)
}
raw_socket_listener/tcp_message.go
+353 -108
View File
@@ -7,13 +7,20 @@ import (
"encoding/hex"
"github.com/buger/gor/proto"
"log"
"net"
"strconv"
"time"
"net"
)
var _ = log.Println
type TCPProtocol uint8
const (
ProtocolHTTP TCPProtocol = 0
ProtocolBinary TCPProtocol = 1
)
// TCPMessage ensure that all TCP packets for given request is received, and processed in right sequence
// Its needed because all TCP message can be fragmented or re-transmitted
//
@@ -36,16 +43,36 @@ type TCPMessage struct {
packets []*TCPPacket
delChan chan *TCPMessage
protocol TCPProtocol
/* HTTP specific variables */
methodType httpMethodType
bodyType httpBodyType
expectType httpExpectType
seqMissing bool
headerPacket int
contentLength int
complete bool
}
// NewTCPMessage pointer created from a Acknowledgment number and a channel of messages readuy to be deleted
func NewTCPMessage(Seq, Ack uint32, IsIncoming bool) (msg *TCPMessage) {
msg = &TCPMessage{Seq: Seq, Ack: Ack, IsIncoming: IsIncoming}
func NewTCPMessage(Seq, Ack uint32, IsIncoming bool, protocol TCPProtocol, timestamp time.Time) (msg *TCPMessage) {
msg = &TCPMessage{Seq: Seq, Ack: Ack, IsIncoming: IsIncoming, protocol: protocol, Start: timestamp}
msg.Start = time.Now()
return
}
func (t *TCPMessage) packetsData() (d [][]byte) {
d = make([][]byte, len(t.packets))
for i, p := range t.packets {
d[i] = p.Data
}
return
}
// Bytes return message content
func (t *TCPMessage) Bytes() (output []byte) {
for _, p := range t.packets {
@@ -57,13 +84,13 @@ func (t *TCPMessage) Bytes() (output []byte) {
// Size returns total body size
func (t *TCPMessage) BodySize() (size int) {
if len(t.packets) == 0 {
if len(t.packets) == 0 || t.headerPacket == -1 {
return 0
}
size += len(proto.Body(t.packets[0].Data))
size += len(proto.Body(t.packets[t.headerPacket].Data))
for _, p := range t.packets[1:] {
for _, p := range t.packets[t.headerPacket+1:] {
size += len(p.Data)
}
@@ -86,145 +113,381 @@ func (t *TCPMessage) Size() (size int) {
// AddPacket to the message and ensure packet uniqueness
// TCP allows that packet can be re-send multiple times
func (t *TCPMessage) AddPacket(packet *TCPPacket) {
packetFound := false
for _, pkt := range t.packets {
if packet.Seq == pkt.Seq {
packetFound = true
break
return
}
}
if !packetFound {
// Packets not always captured in same Seq order, and sometimes we need to prepend
if len(t.packets) == 0 || packet.Seq > t.packets[len(t.packets)-1].Seq {
t.packets = append(t.packets, packet)
} else if packet.Seq < t.packets[0].Seq {
t.packets = append([]*TCPPacket{packet}, t.packets...)
t.Seq = packet.Seq // Message Seq should indicated starting seq
} else { // insert somewhere in the middle...
for i, p := range t.packets {
if packet.Seq < p.Seq {
t.packets = append(t.packets[:i], append([]*TCPPacket{packet}, t.packets[i:]...)...)
break
}
// Packets not always captured in same Seq order, and sometimes we need to prepend
if len(t.packets) == 0 || packet.Seq > t.packets[len(t.packets)-1].Seq {
t.packets = append(t.packets, packet)
} else if packet.Seq < t.packets[0].Seq {
t.packets = append([]*TCPPacket{packet}, t.packets...)
t.Seq = packet.Seq // Message Seq should indicated starting seq
} else { // insert somewhere in the middle...
for i, p := range t.packets {
if packet.Seq < p.Seq {
t.packets = append(t.packets[:i], append([]*TCPPacket{packet}, t.packets[i:]...)...)
break
}
}
}
if t.IsIncoming {
t.End = time.Now()
} else {
t.End = time.Now().Add(time.Millisecond)
}
if t.IsIncoming {
t.End = time.Now()
} else {
t.End = time.Now().Add(time.Millisecond)
}
if packet.OrigAck != 0 {
t.DataAck = packet.OrigAck
}
if packet.OrigAck != 0 {
t.DataAck = packet.OrigAck
}
if packet.timestamp.Before(t.Start) {
t.Start = packet.timestamp
}
t.checkSeqIntegrity()
if t.protocol == ProtocolHTTP {
t.updateHeadersPacket()
t.updateMethodType()
t.updateBodyType()
t.check100Continue()
t.checkIfComplete()
}
}
// Check if there is missing packet
func (t *TCPMessage) isSeqMissing() bool {
func (t *TCPMessage) checkSeqIntegrity() {
if len(t.packets) == 1 {
return false
t.seqMissing = false
}
for i, p := range t.packets {
offset := len(t.packets) - 1
if t.packets[offset].IsFIN {
offset--
if offset < 0 {
return
}
}
for i, p := range t.packets[:offset] {
if p.IsFIN {
continue
}
// If final packet
if len(t.packets) == i+1 {
return false
t.seqMissing = false
return
}
np := t.packets[i+1]
if np.Seq != p.Seq+uint32(len(p.Data)) {
return true
nextSeq := p.Seq + uint32(len(p.Data))
if np.Seq != nextSeq {
if t.protocol == ProtocolHTTP && t.expectType == httpExpect100Continue {
if np.Seq != nextSeq+22 {
t.seqMissing = true
return
}
} else {
t.seqMissing = true
return
}
}
}
return false
t.seqMissing = false
}
var EmptyLine = []byte("\r\n\r\n")
var ChunkEnd = []byte("0\r\n\r\n")
var bEmptyLine = []byte("\r\n\r\n")
var bChunkEnd = []byte("0\r\n\r\n")
func (t *TCPMessage) isHeadersReceived() bool {
for _, p := range t.packets {
if bytes.LastIndex(p.Data, EmptyLine) != -1 {
return true
func (t *TCPMessage) updateHeadersPacket() {
if len(t.packets) == 1 {
t.headerPacket = -1
}
if t.headerPacket != -1 {
return
}
if t.seqMissing {
return
}
for i, p := range t.packets {
if bytes.LastIndex(p.Data, bEmptyLine) != -1 {
t.headerPacket = i
return
}
}
return false
return
}
// isMultipart returns true if message contains from multiple tcp packets
func (t *TCPMessage) IsFinished() bool {
payload := t.packets[0].Data
if len(payload) < 4 {
return true
func (t *TCPMessage) checkIfComplete() {
if t.seqMissing || t.headerPacket == -1 {
return
}
m := payload[:4]
if t.methodType == httpMethodNotFound {
return
}
// Responses can be emitted only if we found request
if !t.IsIncoming && t.AssocMessage == nil {
return
}
// If one GET, OPTIONS, or HEAD request
if t.methodType == httpMethodWithoutBody {
t.complete = true
} else {
switch t.bodyType {
case httpBodyEmpty:
t.complete = true
case httpBodyContentLength:
if t.contentLength == 0 || t.contentLength == t.BodySize() {
t.complete = true
}
case httpBodyChunked:
lastPacket := t.packets[len(t.packets)-1]
if bytes.LastIndex(lastPacket.Data, bChunkEnd) != -1 {
t.complete = true
}
default:
if len(t.packets) == 0 {
return
}
last := t.packets[len(t.packets)-1]
if last.IsFIN {
t.complete = true
}
}
}
}
type httpMethodType uint8
const (
httpMethodNotSet httpMethodType = 0
httpMethodWithBody httpMethodType = 1
httpMethodWithoutBody httpMethodType = 2
httpMethodNotFound httpMethodType = 3
)
var methodsWithBody = [][]byte{
[]byte("POST"),
[]byte("PUT"),
[]byte("PATCH"),
[]byte("CONNECT"),
}
func (t *TCPMessage) updateMethodType() {
// if there is cache
if t.methodType != httpMethodNotSet && t.methodType != httpMethodNotFound {
return
}
d := t.packets[0].Data
// Minimum length fo request: GET / HTTP/1.1\r\n
if len(d) < 16 {
t.methodType = httpMethodNotFound
return
}
if t.IsIncoming {
// If one GET, OPTIONS, or HEAD request
if bytes.Equal(m, []byte("GET ")) || bytes.Equal(m, []byte("OPTI")) || bytes.Equal(m, []byte("HEAD")) {
if !t.isSeqMissing() && t.isHeadersReceived() {
return true
} else {
return false
var method []byte
if mIdx := bytes.IndexByte(d[:8], ' '); mIdx != -1 {
method = d[:mIdx]
// Check that after method we have absolute or relative path
switch d[mIdx+1] {
case '/', 'h', '*':
default:
t.methodType = httpMethodNotFound
return
}
} else {
// Sometimes header comes after the body :(
if bytes.Equal(m, []byte("POST")) || bytes.Equal(m, []byte("PUT ")) || bytes.Equal(m, []byte("PATC")) {
t.methodType = httpMethodNotFound
return
}
if t.isHeadersReceived() {
if length := proto.Header(payload, []byte("Content-Length")); len(length) > 0 {
l, _ := strconv.Atoi(string(length))
// If content-length equal current body length
if l > 0 && l == t.BodySize() {
return true
}
}
}
for _, m := range methodsWithBody {
if len(m) == len(method) && bytes.Equal(m, method) {
t.methodType = httpMethodWithBody
return
}
}
t.methodType = httpMethodWithoutBody
} else {
// Request not found
// Can be because response came first or request request was just missing
if t.AssocMessage == nil {
if !bytes.Equal(d[:6], []byte("HTTP/1")) {
t.methodType = httpMethodNotFound
return
}
t.methodType = httpMethodWithBody
}
}
type httpBodyType uint8
const (
httpBodyNotSet httpBodyType = 0
httpBodyEmpty httpBodyType = 1
httpBodyContentLength httpBodyType = 2
httpBodyChunked httpBodyType = 3
httpBodyConnectionClose httpBodyType = 4
)
func (t *TCPMessage) updateBodyType() {
// if there is cache
if t.bodyType != httpBodyNotSet {
return
}
// Headers not received
if t.headerPacket == -1 {
return
}
var lengthB, encB, connB []byte
proto.ParseHeaders(t.packetsData(), func(header, value []byte)bool{
if proto.HeadersEqual(header, []byte("Content-Length")) {
lengthB = value
return false
}
if !bytes.Equal(m, []byte("HTTP")) {
if proto.HeadersEqual(header, []byte("Transfer-Encoding")) {
encB = value
return false
}
if length := proto.Header(payload, []byte("Content-Length")); len(length) > 0 {
if length[0] == '0' {
return true
}
if proto.HeadersEqual(header, []byte("Connection")) {
connB = value
}
l, _ := strconv.Atoi(string(length))
return true
})
// If content-length equal current body length
if l > 0 && l == t.BodySize() {
return true
}
} else {
if enc := proto.Header(payload, []byte("Transfer-Encoding")); len(enc) == 0 {
return true
switch t.methodType {
case httpMethodNotFound:
return
case httpMethodWithoutBody:
t.bodyType = httpBodyEmpty
return
case httpMethodWithBody:
if len(lengthB) > 0 {
t.contentLength, _ = strconv.Atoi(string(lengthB))
if t.contentLength == 0 {
t.bodyType = httpBodyEmpty
} else {
if len(t.packets) > 1 && bytes.LastIndex(t.packets[len(t.packets)-1].Data, ChunkEnd) != -1 {
return true
}
t.bodyType = httpBodyContentLength
}
return
}
if len(encB) > 0 {
t.bodyType = httpBodyChunked
return
}
if len(connB) > 0 && bytes.Equal(connB, []byte("close")) {
t.bodyType = httpBodyConnectionClose
return
}
}
return false
t.bodyType = httpBodyEmpty
}
type httpExpectType uint8
const (
httpExpectNotSet httpExpectType = 0
httpExpectEmpty httpExpectType = 1
httpExpect100Continue httpExpectType = 2
)
var bExpectHeader = []byte("Expect")
var bExpect100Value = []byte("100-continue")
func (t *TCPMessage) check100Continue() {
if t.expectType != httpExpectNotSet || len(t.packets[0].Data) < 25 {
return
}
if t.methodType != httpMethodWithBody {
return
}
if t.seqMissing || t.headerPacket == -1 {
return
}
last := t.packets[len(t.packets)-1]
// reading last 4 bytes for double CRLF
if !bytes.HasSuffix(last.Data, bEmptyLine) {
return
}
var expectB []byte
proto.ParseHeaders(t.packetsData(), func(header, value []byte)bool{
if proto.HeadersEqual(header, bExpectHeader) {
expectB = value
return false
}
return true
})
if len(expectB) > 0 && bytes.Equal(bExpect100Value, expectB) {
t.expectType = httpExpect100Continue
return
}
t.expectType = httpExpectEmpty
}
func (t *TCPMessage) setAssocMessage(m *TCPMessage) {
t.AssocMessage = m
t.checkIfComplete()
}
// UpdateResponseAck should be called after packet is added
func (t *TCPMessage) UpdateResponseAck() uint32 {
lastPacket := t.packets[len(t.packets)-1]
if lastPacket.IsFIN && len(t.packets) > 1 {
lastPacket = t.packets[len(t.packets)-2]
}
respAck := lastPacket.Seq + uint32(len(lastPacket.Data))
if t.ResponseAck != respAck {
t.ResponseAck = lastPacket.Seq + uint32(len(lastPacket.Data))
// We swappwed src and dst port
copy(t.ResponseID[:16], lastPacket.Addr)
copy(t.ResponseID[16:], lastPacket.Raw[2:4]) // Src port
copy(t.ResponseID[18:], lastPacket.Raw[0:2]) // Dest port
binary.BigEndian.PutUint32(t.ResponseID[20:24], t.ResponseAck)
}
return t.ResponseAck
}
func (t *TCPMessage) UUID() []byte {
@@ -247,28 +510,10 @@ func (t *TCPMessage) UUID() []byte {
return uuid
}
// UpdateResponseAck should be called after packet is added
func (t *TCPMessage) UpdateResponseAck() uint32 {
lastPacket := t.packets[len(t.packets)-1]
respAck := lastPacket.Seq + uint32(len(lastPacket.Data))
if t.ResponseAck != respAck {
t.ResponseAck = lastPacket.Seq + uint32(len(lastPacket.Data))
// We swappwed src and dst port
copy(t.ResponseID[:16], lastPacket.Addr)
copy(t.ResponseID[16:], lastPacket.Raw[2:4]) // Src port
copy(t.ResponseID[18:], lastPacket.Raw[0:2]) // Dest port
binary.BigEndian.PutUint32(t.ResponseID[20:24], t.ResponseAck)
}
return t.ResponseAck
}
func (t *TCPMessage) ID() tcpID {
return t.packets[0].ID
}
func (t *TCPMessage) IP() net.IP {
return net.IP(t.packets[0].Addr)
}
}
raw_socket_listener/tcp_message_test.go
+155 -88
View File
@@ -5,9 +5,10 @@ import (
"encoding/binary"
_ "log"
"testing"
"time"
)
func buildPacket(isIncoming bool, Ack, Seq uint32, Data []byte) (packet *TCPPacket) {
func buildPacket(isIncoming bool, Ack, Seq uint32, Data []byte, timestamp time.Time) (packet *TCPPacket) {
var srcPort, destPort uint16
// For tests `listening` port is 0
@@ -25,7 +26,7 @@ func buildPacket(isIncoming bool, Ack, Seq uint32, Data []byte) (packet *TCPPack
buf[12] = 64
buf = append(buf, Data...)
packet = ParseTCPPacket([]byte("123"), buf)
packet = ParseTCPPacket([]byte("123"), buf, timestamp)
return packet
}
@@ -36,31 +37,31 @@ func buildMessage(p *TCPPacket) *TCPMessage {
isIncoming = true
}
m := NewTCPMessage(p.Seq, p.Ack, isIncoming)
m := NewTCPMessage(p.Seq, p.Ack, isIncoming, ProtocolHTTP)
m.AddPacket(p)
return m
}
func TestTCPMessagePacketsOrder(t *testing.T) {
msg := buildMessage(buildPacket(true, 1, 1, []byte("a")))
msg.AddPacket(buildPacket(true, 1, 2, []byte("b")))
msg := buildMessage(buildPacket(true, 1, 1, []byte("a"), time.Now()))
msg.AddPacket(buildPacket(true, 1, 2, []byte("b"), time.Now()))
if !bytes.Equal(msg.Bytes(), []byte("ab")) {
t.Error("Should contatenate packets in right order")
}
// When first packet have wrong order (Seq)
msg = buildMessage(buildPacket(true, 1, 2, []byte("b")))
msg.AddPacket(buildPacket(true, 1, 1, []byte("a")))
msg = buildMessage(buildPacket(true, 1, 2, []byte("b"), time.Now()))
msg.AddPacket(buildPacket(true, 1, 1, []byte("a"), time.Now()))
if !bytes.Equal(msg.Bytes(), []byte("ab")) {
t.Error("Should contatenate packets in right order")
}
// Should ignore packets with same sequence
msg = buildMessage(buildPacket(true, 1, 1, []byte("a")))
msg.AddPacket(buildPacket(true, 1, 1, []byte("a")))
msg = buildMessage(buildPacket(true, 1, 1, []byte("a"), time.Now()))
msg.AddPacket(buildPacket(true, 1, 1, []byte("a"), time.Now()))
if !bytes.Equal(msg.Bytes(), []byte("a")) {
t.Error("Should ignore packet with same Seq")
@@ -68,8 +69,8 @@ func TestTCPMessagePacketsOrder(t *testing.T) {
}
func TestTCPMessageSize(t *testing.T) {
msg := buildMessage(buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\na")))
msg.AddPacket(buildPacket(true, 1, 2, []byte("b")))
msg := buildMessage(buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\na"), time.Now()))
msg.AddPacket(buildPacket(true, 1, 2, []byte("b"), time.Now()))
if msg.BodySize() != 2 {
t.Error("Should count only body", msg.BodySize())
@@ -80,112 +81,178 @@ func TestTCPMessageSize(t *testing.T) {
}
}
func TestTCPMessageIsFinished(t *testing.T) {
methodsWithoutBodies := []string{"GET", "OPTIONS", "HEAD"}
func TestTCPMessageIsComplete(t *testing.T) {
testCases := []struct {
direction bool
payload string
assocMessage bool
expectedCompleted bool
}{
{true, "GET / HTTP/1.1\r\n\r\n", false, true},
{true, "HEAD / HTTP/1.1\r\n\r\n", false, true},
{false, "HTTP/1.1 200 OK\r\n\r\n", true, true},
{true, "POST / HTTP/1.1\r\nContent-Length: 1\r\n\r\na", false, true},
{true, "PUT / HTTP/1.1\r\nContent-Length: 1\r\n\r\na", false, true},
{false, "HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n", true, true},
{false, "HTTP/1.1 200 OK\r\nContent-Length: 1\r\n\r\na", true, true},
{false, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n0\r\n\r\n", true, true},
for _, m := range methodsWithoutBodies {
msg := buildMessage(buildPacket(true, 1, 1, []byte(m+" / HTTP/1.1\r\n\r\n")))
// chunked not finished
{false, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n", true, false},
if !msg.IsFinished() {
t.Error(m, " request should be finished")
// content-length != actual length
{true, "POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\na", false, false},
{false, "HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\na", true, false},
// non-valid http request
{true, "UNKNOWN asd HTTP/1.1\r\n\r\n", false, false},
// response without associated request
{false, "HTTP/1.1 200 OK\r\n\r\n", false, false},
}
for _, tc := range testCases {
msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payload), time.Now()))
if tc.assocMessage {
msg.AssocMessage = &TCPMessage{}
}
}
msg.checkIfComplete()
methodsWithBodies := []string{"POST", "PUT", "PATCH"}
for _, m := range methodsWithBodies {
msg := buildMessage(buildPacket(true, 1, 1, []byte(m+" / HTTP/1.1\r\nContent-Length: 1\r\n\r\na")))
if !msg.IsFinished() {
t.Error(m, " should be finished as body length == content length")
if msg.complete != tc.expectedCompleted {
t.Errorf("Payload %s: Expected %t, got %t.", tc.payload, tc.expectedCompleted, msg.complete)
}
msg = buildMessage(buildPacket(true, 1, 1, []byte(m+" / HTTP/1.1\r\nContent-Length: 2\r\n\r\na")))
if msg.IsFinished() {
t.Error(m, " should not be finished as body length != content length")
}
}
msg := buildMessage(buildPacket(true, 1, 1, []byte("UNKNOWN / HTTP/1.1\r\n\r\n")))
if msg.IsFinished() {
t.Error("non http or wrong methods considered as not finished")
}
// Responses
msg = buildMessage(buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\n\r\n")))
msg.AssocMessage = &TCPMessage{}
if !msg.IsFinished() {
t.Error("Should mark simple response as finished")
}
msg = buildMessage(buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\n\r\n")))
msg.AssocMessage = nil
if msg.IsFinished() {
t.Error("Should not mark responses without associated requests")
}
msg = buildMessage(buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n")))
msg.AssocMessage = &TCPMessage{}
if msg.IsFinished() {
t.Error("Should mark chunked response as non finished")
}
msg = buildMessage(buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")))
msg.AssocMessage = &TCPMessage{}
if !msg.IsFinished() {
t.Error("Should mark Content-Length: 0 respones as finished")
}
msg = buildMessage(buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\nContent-Length: 1\r\n\r\na")))
msg.AssocMessage = &TCPMessage{}
if !msg.IsFinished() {
t.Error("Should mark valid Content-Length respones as finished")
}
msg = buildMessage(buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\na")))
msg.AssocMessage = &TCPMessage{}
if msg.IsFinished() {
t.Error("Should not mark not valid Content-Length respones as finished")
}
}
func TestTCPMessageIsSeqMissing(t *testing.T) {
p1 := buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\n"))
p2 := buildPacket(false, 1, p1.Seq+uint32(len(p1.Data)), []byte("Content-Length: 10\r\n\r\n"))
p3 := buildPacket(false, 1, p2.Seq+uint32(len(p2.Data)), []byte("a"))
p1 := buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\n"), time.Now())
p2 := buildPacket(false, 1, p1.Seq+uint32(len(p1.Data)), []byte("Content-Length: 10\r\n\r\n"), time.Now())
p3 := buildPacket(false, 1, p2.Seq+uint32(len(p2.Data)), []byte("a"), time.Now())
msg := buildMessage(p1)
if msg.isSeqMissing() {
if msg.seqMissing {
t.Error("Should be complete if have only 1 packet")
}
msg.AddPacket(p3)
if !msg.isSeqMissing() {
if !msg.seqMissing {
t.Error("Should be incomplete because missing middle component")
}
msg.AddPacket(p2)
if msg.isSeqMissing() {
if msg.seqMissing {
t.Error("Should be complete once missing packet added")
}
}
func TestTCPMessageIsHeadersReceived(t *testing.T) {
p1 := buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\n"))
p2 := buildPacket(false, 1, p1.Seq+uint32(len(p1.Data)), []byte("Content-Length: 10\r\n\r\n"))
p1 := buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now())
p2 := buildPacket(false, 1, p1.Seq+uint32(len(p1.Data)), []byte("Content-Length: 10\r\n\r\n"), time.Now())
msg := buildMessage(p1)
if msg.isHeadersReceived() {
t.Error("Should be complete if have only 1 packet")
if msg.headerPacket == -1 {
t.Error("Should be complete if have only 1 packet", msg.headerPacket)
}
msg.AddPacket(p2)
if !msg.isHeadersReceived() {
if msg.headerPacket == -1 {
t.Error("Should found double new line: headers received")
}
msg = buildMessage(buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\nContent-Length: 1\r\n"), time.Now()))
if msg.headerPacket != -1 {
t.Error("Should not find headers end")
}
}
func TestTCPMessageMethodType(t *testing.T) {
testCases := []struct {
direction bool
payload string
expectedMethodType httpMethodType
}{
{true, "GET / HTTP/1.1\r\n\r\n", httpMethodWithoutBody},
{true, "GET * HTTP/1.1\r\n\r\n", httpMethodWithoutBody},
{true, "UNKNOWN / HTTP/1.1\r\n\r\n", httpMethodWithoutBody},
{true, "GET http://example.com HTTP/1.1\r\n\r\n", httpMethodWithoutBody},
{true, "POST / HTTP/1.1\r\n\r\n", httpMethodWithBody},
{true, "PUT / HTTP/1.1\r\n\r\n", httpMethodWithBody},
{true, "GET zxc HTTP/1.1\r\n\r\n", httpMethodNotFound},
{true, "GET / HTTP\r\n\r\n", httpMethodNotFound},
{true, "VERYLONGMETHOD / HTTP/1.1\r\n\r\n", httpMethodNotFound},
{false, "HTTP/1.1 200 OK\r\n\r\n", httpMethodWithBody},
{false, "HTTP /1.1 200 OK\r\n\r\n", httpMethodNotFound},
}
for _, tc := range testCases {
msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payload), time.Now()))
if msg.methodType != tc.expectedMethodType {
t.Errorf("Expected %d, got %d", tc.expectedMethodType, msg.methodType)
}
}
}
func TestTCPMessageBodyType(t *testing.T) {
testCases := []struct {
direction bool
payload string
expectedBodyType httpBodyType
}{
{true, "GET / HTTP/1.1\r\n\r\n", httpBodyEmpty},
{true, "POST / HTTP/1.1\r\n\r\n", httpBodyEmpty},
{true, "POST / HTTP/1.1\r\nUser-Agent: zxc\r\n\r\n", httpBodyEmpty},
{false, "HTTP/1.1 200 OK\r\n\r\n", httpBodyEmpty},
{true, "POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab", httpBodyContentLength},
{false, "HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nab", httpBodyContentLength},
{true, "POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n2\r\nab\r\n0\r\n\r\n", httpBodyChunked},
{false, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n2\r\nab\r\n0\r\n\r\n", httpBodyChunked},
}
for _, tc := range testCases {
msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payload), time.Now()))
if msg.bodyType != tc.expectedBodyType {
t.Errorf("Expected %d, got %d", tc.expectedBodyType, msg.bodyType)
}
}
}
func TestTCPMessageBodySize(t *testing.T) {
testCases := []struct {
direction bool
payloads []string
expectedSize int
}{
{true, []string{"GET / HTTP/1.1\r\n\r\n"}, 0},
{true, []string{"POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab"}, 2},
{true, []string{"GET / HTTP/1.1\r\n", "Content-Length: 2\r\n\r\nab"}, 2},
{true, []string{"GET / HTTP/1.1\r\n", "Content-Length: 2\r\n\r\n", "ab"}, 2},
}
for _, tc := range testCases {
msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payloads[0]), time.Now()))
if len(tc.payloads) > 1 {
for _, p := range tc.payloads[1:] {
seq := uint32(1 + msg.Size())
msg.AddPacket(buildPacket(tc.direction, 1, seq, []byte(p), time.Now()))
}
}
if msg.BodySize() != tc.expectedSize {
t.Errorf("Expected %d, got %d", tc.expectedSize, msg.BodySize())
}
}
}
func TestTcpMessageStart(t *testing.T) {
start := time.Now().Add(-1 * time.Second)
msg := buildMessage(buildPacket(true, 1, 2, []byte("b"), time.Now()))
msg.AddPacket(buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\na"), start))
if msg.Start != start {
t.Error("Message timestamp should be equal to the lowest related packet timestamp", start, msg.Start)
}
}
raw_socket_listener/tcp_packet.go
+29 -12
View File
@@ -5,6 +5,7 @@ import (
"log"
"strconv"
"strings"
"time"
)
var _ = log.Println
@@ -33,18 +34,21 @@ type TCPPacket struct {
Ack uint32
OrigAck uint32
DataOffset uint8
IsFIN bool
Raw []byte
Data []byte
Addr []byte
timestamp time.Time
ID tcpID
}
// ParseTCPPacket takes address and tcp payload and returns parsed TCPPacket
func ParseTCPPacket(addr []byte, data []byte) (p *TCPPacket) {
func ParseTCPPacket(addr []byte, data []byte, timestamp time.Time) (p *TCPPacket) {
p = &TCPPacket{Raw: data}
p.ParseBasic()
p.Addr = addr
p.timestamp = timestamp
p.GenID()
return
@@ -71,28 +75,40 @@ func (t *TCPPacket) ParseBasic() {
t.Seq = binary.BigEndian.Uint32(t.Raw[4:8])
t.Ack = binary.BigEndian.Uint32(t.Raw[8:12])
t.DataOffset = (t.Raw[12] & 0xF0) >> 4
t.IsFIN = t.Raw[13]&0x01 != 0
// log.Println("DataOffset:", t.DataOffset, t.DestPort, t.SrcPort, t.Seq, t.Ack)
t.Data = t.Raw[t.DataOffset*4:]
}
func (t *TCPPacket) Dump() []byte {
buf := make([]byte, len(t.Data)+16+16)
copy(buf[:16], t.Addr)
func (t *TCPPacket) dump() *packet {
tcpBuf := buf[16:]
packetSrcIP := make([]byte, 16)
packetData := make([]byte, len(t.Data) + 16)
binary.BigEndian.PutUint16(tcpBuf[2:4], t.DestPort)
binary.BigEndian.PutUint16(tcpBuf[0:2], t.SrcPort)
copy(packetSrcIP, t.Addr)
binary.BigEndian.PutUint32(tcpBuf[4:8], t.Seq)
binary.BigEndian.PutUint32(tcpBuf[8:12], t.Ack)
binary.BigEndian.PutUint16(packetData[0:2], t.SrcPort)
binary.BigEndian.PutUint16(packetData[2:4], t.DestPort)
tcpBuf[12] = 64
copy(tcpBuf[16:], t.Data)
binary.BigEndian.PutUint32(packetData[4:8], t.Seq)
binary.BigEndian.PutUint32(packetData[8:12], t.Ack)
packetData[12] = 64
if t.IsFIN {
packetData[13] = packetData[13] | 0x01
}
copy(packetData[16:], t.Data)
return &packet{
srcIP: packetSrcIP,
data:packetData,
timestamp:t.timestamp,
}
return buf
}
// String output for a TCP Packet
@@ -109,6 +125,7 @@ func (t *TCPPacket) String() string {
"Sequence:" + strconv.Itoa(int(t.Seq)),
"Acknowledgment:" + strconv.Itoa(int(t.Ack)),
"Header len:" + strconv.Itoa(int(t.DataOffset)),
"FIN:" + strconv.FormatBool(t.IsFIN),
"Data size:" + strconv.Itoa(len(t.Data)),
"Data:" + string(t.Data[:maxLen]),
s3/index.html
+60
View File
File diff suppressed because one or more lines are too long
settings.go
+40 -8
View File
@@ -25,15 +25,18 @@ func (h *MultiOption) Set(value string) error {
// AppSettings is the struct of main configuration
type AppSettings struct {
verbose bool
debug bool
stats bool
verbose bool
debug bool
stats bool
exitAfter time.Duration
splitOutput bool
recognizeTCPSessions bool
inputDummy MultiOption
outputDummy MultiOption
outputStdout bool
outputNull bool
inputTCP MultiOption
outputTCP MultiOption
@@ -51,14 +54,21 @@ type AppSettings struct {
inputRAWEngine string
inputRAWTrackResponse bool
inputRAWRealIPHeader string
inputRAWProtocol string
middleware string
inputHTTP MultiOption
outputHTTP MultiOption
outputHTTP MultiOption
outputHTTPConfig HTTPOutputConfig
outputBinary MultiOption
outputBinaryConfig BinaryOutputConfig
modifierConfig HTTPModifierConfig
outputKafkaConfig KafkaConfig
}
// Settings holds Gor configuration
@@ -76,14 +86,19 @@ func init() {
flag.BoolVar(&Settings.verbose, "verbose", false, "Turn on more verbose output")
flag.BoolVar(&Settings.debug, "debug", false, "Turn on debug output, shows all intercepted traffic. Works only when with `verbose` flag")
flag.BoolVar(&Settings.stats, "stats", false, "Turn on queue stats output")
flag.DurationVar(&Settings.exitAfter, "exit-after", 0, "exit after specified duration")
flag.BoolVar(&Settings.splitOutput, "split-output", false, "By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs.")
flag.BoolVar(&Settings.recognizeTCPSessions, "recognize-tcp-sessions", false, "[PRO] If turned on http output will create separate worker for each TCP session. Splitting output will session based as well.")
flag.Var(&Settings.inputDummy, "input-dummy", "Used for testing outputs. Emits 'Get /' request every 1s")
flag.Var(&Settings.outputDummy, "output-dummy", "DEPRECATED: use --output-stdout instead")
flag.BoolVar(&Settings.outputStdout, "output-stdout", false, "Used for testing inputs. Just prints to console data coming from inputs.")
flag.BoolVar(&Settings.outputNull, "output-null", false, "Used for testing inputs. Drops all requests.")
flag.Var(&Settings.inputTCP, "input-tcp", "Used for internal communication between Gor instances. Example: \n\t# Receive requests from other Gor instances on 28020 port, and redirect output to staging\n\tgor --input-tcp :28020 --output-http staging.com")
flag.Var(&Settings.outputTCP, "output-tcp", "Used for internal communication between Gor instances. Example: \n\t# Listen for requests on 80 port and forward them to other Gor instance on 28020 port\n\tgor --input-raw :80 --output-tcp replay.local:28020")
flag.BoolVar(&Settings.outputTCPStats, "output-tcp-stats", false, "Report TCP output queue stats to console every 5 seconds.")
@@ -111,23 +126,40 @@ func init() {
flag.StringVar(&Settings.inputRAWEngine, "input-raw-engine", "libpcap", "Intercept traffic using `libpcap` (default), and `raw_socket`")
flag.StringVar(&Settings.inputRAWProtocol, "input-raw-protocol", "http", "Specify application protocol of intercepted traffic. Possible values: http, binary")
flag.StringVar(&Settings.inputRAWRealIPHeader, "input-raw-realip-header", "", "If not blank, injects header with given name and real IP value to the request payload. Usually this header should be named: X-Real-IP")
flag.StringVar(&Settings.middleware, "middleware", "", "Used for modifying traffic using external command")
flag.Var(&Settings.inputHTTP, "input-http", "Read requests from HTTP, should be explicitly sent from your application:\n\t# Listen for http on 9000\n\tgor --input-http :9000 --output-http staging.com")
// flag.Var(&Settings.inputHTTP, "input-http", "Read requests from HTTP, should be explicitly sent from your application:\n\t# Listen for http on 9000\n\tgor --input-http :9000 --output-http staging.com")
flag.Var(&Settings.outputHTTP, "output-http", "Forwards incoming requests to given http address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --output-http http://staging.com")
/* outputHTTPConfig */
flag.IntVar(&Settings.outputHTTPConfig.BufferSize, "output-http-response-buffer", 0, "HTTP response buffer size, all data after this size will be discarded.")
flag.IntVar(&Settings.outputHTTPConfig.workers, "output-http-workers", 0, "Gor uses dynamic worker scaling by default. Enter a number to run a set number of workers.")
flag.IntVar(&Settings.outputHTTPConfig.redirectLimit, "output-http-redirects", 0, "Enable how often redirects should be followed.")
flag.DurationVar(&Settings.outputHTTPConfig.Timeout, "output-http-timeout", 0, "Specify HTTP request/response timeout. By default 5s. Example: --output-http-timeout 30s")
flag.DurationVar(&Settings.outputHTTPConfig.Timeout, "output-http-timeout", 5*time.Second, "Specify HTTP request/response timeout. By default 5s. Example: --output-http-timeout 30s")
flag.BoolVar(&Settings.outputHTTPConfig.stats, "output-http-stats", false, "Report http output queue stats to console every 5 seconds.")
flag.BoolVar(&Settings.outputHTTPConfig.OriginalHost, "http-original-host", false, "Normally gor replaces the Host http header with the host supplied with --output-http. This option disables that behavior, preserving the original Host header.")
flag.BoolVar(&Settings.outputHTTPConfig.Debug, "output-http-debug", false, "Enables http debug output.")
flag.StringVar(&Settings.outputHTTPConfig.elasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'")
/* outputHTTPConfig */
flag.Var(&Settings.outputBinary, "output-binary", "Forwards incoming binary payloads to given address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --input-raw-protocol binary --output-binary staging.com:80")
/* outputBinaryConfig */
flag.IntVar(&Settings.outputBinaryConfig.BufferSize, "output-tcp-response-buffer", 0, "TCP response buffer size, all data after this size will be discarded.")
flag.IntVar(&Settings.outputBinaryConfig.workers, "output-binary-workers", 0, "Gor uses dynamic worker scaling by default. Enter a number to run a set number of workers.")
flag.DurationVar(&Settings.outputBinaryConfig.Timeout, "output-binary-timeout", 0, "Specify HTTP request/response timeout. By default 5s. Example: --output-binary-timeout 30s")
flag.BoolVar(&Settings.outputBinaryConfig.Debug, "output-binary-debug", false, "Enables binary debug output.")
/* outputBinaryConfig */
flag.StringVar(&Settings.outputKafkaConfig.host, "output-kafka-host", "", "Send request and response stats to Kafka:\n\tgor --input-raw :8080 --output-kafka-host '192.168.0.1:9092,192.168.0.2:9092'")
flag.StringVar(&Settings.outputKafkaConfig.topic, "output-kafka-topic", "", "Send request and response stats to Kafka:\n\tgor --input-raw :8080 --output-kafka-topic 'kafka-log'")
flag.Var(&Settings.modifierConfig.headers, "http-set-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'")
flag.Var(&Settings.modifierConfig.headers, "output-http-header", "WARNING: `--output-http-header` DEPRECATED, use `--http-set-header` instead")
@@ -148,7 +180,7 @@ func init() {
flag.Var(&Settings.modifierConfig.headerFilters, "http-allow-header", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^v1")
flag.Var(&Settings.modifierConfig.headerFilters, "output-http-header-filter", "WARNING: `--output-http-header-filter` DEPRECATED, use `--http-allow-header` instead")
flag.Var(&Settings.modifierConfig.headerFilters, "http-disallow-header", "A regexp to match a specific header against. Requests with matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-header \"User-Agent: Replayed by Gor\"")
flag.Var(&Settings.modifierConfig.headerNegativeFilters, "http-disallow-header", "A regexp to match a specific header against. Requests with matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-header \"User-Agent: Replayed by Gor\"")
flag.Var(&Settings.modifierConfig.headerHashFilters, "http-header-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:\n\t gor --input-raw :8080 --output-http staging.com --http-header-imiter user-id:25%")
flag.Var(&Settings.modifierConfig.headerHashFilters, "output-http-header-hash-filter", "WARNING: `output-http-header-hash-filter` DEPRECATED, use `--http-header-hash-limiter` instead")
tcp_client.go
+190
View File
@@ -0,0 +1,190 @@
package main
import (
"crypto/tls"
"io"
"log"
"net"
"runtime/debug"
"syscall"
"time"
)
type TCPClientConfig struct {
Debug bool
ConnectionTimeout time.Duration
Timeout time.Duration
ResponseBufferSize int
Secure bool
}
type TCPClient struct {
baseURL string
addr string
conn net.Conn
respBuf []byte
config *TCPClientConfig
redirectsCount int
}
func NewTCPClient(addr string, config *TCPClientConfig) *TCPClient {
if config.Timeout.Nanoseconds() == 0 {
config.Timeout = 5 * time.Second
}
config.ConnectionTimeout = config.Timeout
if config.ResponseBufferSize == 0 {
config.ResponseBufferSize = 100 * 1024 // 100kb
}
client := &TCPClient{config: config, addr: addr}
client.respBuf = make([]byte, config.ResponseBufferSize)
return client
}
func (c *TCPClient) Connect() (err error) {
c.Disconnect()
c.conn, err = net.DialTimeout("tcp", c.addr, c.config.ConnectionTimeout)
if c.config.Secure {
tlsConn := tls.Client(c.conn, &tls.Config{InsecureSkipVerify: true})
if err = tlsConn.Handshake(); err != nil {
return
}
c.conn = tlsConn
}
return
}
func (c *TCPClient) Disconnect() {
if c.conn != nil {
c.conn.Close()
c.conn = nil
Debug("[TCPClient] Disconnected: ", c.baseURL)
}
}
func (c *TCPClient) isAlive() bool {
one := make([]byte, 1)
// Ready 1 byte from socket without timeout to check if it not closed
c.conn.SetReadDeadline(time.Now().Add(time.Millisecond))
_, err := c.conn.Read(one)
if err == nil {
return true
} else if err == io.EOF {
if c.config.Debug {
Debug("[TCPClient] connection closed, reconnecting")
}
return false
} else if err == syscall.EPIPE {
Debug("Detected broken pipe.", err)
return false
}
return true
}
func (c *TCPClient) Send(data []byte) (response []byte, err error) {
// Don't exit on panic
defer func() {
if r := recover(); r != nil {
Debug("[TCPClient]", r, string(data))
if _, ok := r.(error); !ok {
log.Println("[TCPClient] Failed to send request: ", string(data))
log.Println("PANIC: pkg:", r, debug.Stack())
}
}
}()
if c.conn == nil || !c.isAlive() {
Debug("[TCPClient] Connecting:", c.baseURL)
if err = c.Connect(); err != nil {
log.Println("[TCPClient] Connection error:", err)
return
}
}
timeout := time.Now().Add(c.config.Timeout)
c.conn.SetWriteDeadline(timeout)
if c.config.Debug {
Debug("[TCPClient] Sending:", string(data))
}
if _, err = c.conn.Write(data); err != nil {
Debug("[TCPClient] Write error:", err, c.baseURL)
return
}
var readBytes, n int
var currentChunk []byte
timeout = time.Now().Add(c.config.Timeout)
for {
c.conn.SetReadDeadline(timeout)
if readBytes < len(c.respBuf) {
n, err = c.conn.Read(c.respBuf[readBytes:])
readBytes += n
if err != nil {
if err == io.EOF {
err = nil
}
break
}
} else {
if currentChunk == nil {
currentChunk = make([]byte, readChunkSize)
}
n, err = c.conn.Read(currentChunk)
if err == io.EOF {
break
} else if err != nil {
Debug("[TCPClient] Read the whole body error:", err, c.baseURL)
break
}
readBytes += int(n)
}
if readBytes >= maxResponseSize {
Debug("[TCPClient] Body is more than the max size", maxResponseSize,
c.baseURL)
break
}
// For following chunks expect less timeout
timeout = time.Now().Add(c.config.Timeout / 5)
}
if err != nil {
Debug("[TCPClient] Response read error", err, c.conn, readBytes)
return
}
if readBytes > len(c.respBuf) {
readBytes = len(c.respBuf)
}
payload := make([]byte, readBytes)
copy(payload, c.respBuf[:readBytes])
if c.config.Debug {
Debug("[TCPClient] Received:", string(payload))
}
return payload, err
}
test_input.go
+16 -4
View File
@@ -9,6 +9,7 @@ import (
// TestInput used for testing purpose, it allows emitting requests on demand
type TestInput struct {
data chan []byte
disableHeaders bool
}
// NewTestInput constructor for TestInput
@@ -22,11 +23,21 @@ func NewTestInput() (i *TestInput) {
func (i *TestInput) Read(data []byte) (int, error) {
buf := <-i.data
header := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano())
copy(data[0:len(header)], header)
copy(data[len(header):], buf)
if !i.disableHeaders {
header := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
copy(data[0:len(header)], header)
copy(data[len(header):], buf)
return len(buf) + len(header), nil
return len(buf) + len(header), nil
} else {
copy(data, buf)
return len(buf), nil
}
}
// EmitGET emits GET request without headers
func (i *TestInput) EmitBytes(b []byte) {
i.data <- b
}
// EmitGET emits GET request without headers
@@ -34,6 +45,7 @@ func (i *TestInput) EmitGET() {
i.data <- []byte("GET / HTTP/1.1\r\n\r\n")
}
// EmitPOST emits POST request with Content-Length
func (i *TestInput) EmitPOST() {
i.data <- []byte("POST /pub/WWW/ HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
vendor/github.com/araddon/gou
Generated Vendored Submodule
+1
Submodule vendor/github.com/araddon/gou added at 50a94aa4a3
vendor/github.com/bitly/go-hostpool
Generated Vendored Submodule
+1
Submodule vendor/github.com/bitly/go-hostpool added at d0e59c22a5
vendor/github.com/bmizerany/assert
Generated Vendored Submodule
+1
Submodule vendor/github.com/bmizerany/assert added at b7ed37b828
vendor/github.com/google/gopacket
Generated Vendored Submodule
+1
Submodule vendor/github.com/google/gopacket added at c4d647984d
vendor/github.com/mattbaird/elastigo
Generated Vendored Submodule
+1
Submodule vendor/github.com/mattbaird/elastigo added at 34c4c4d842