1use super::recv::RecvHeaderBlockError;
2use super::store::{self, Entry, Resolve, Store};
3use super::{Buffer, Config, Counts, Prioritized, Recv, Send, Stream, StreamId};
4use crate::codec::{Codec, SendError, UserError};
5use crate::ext::Protocol;
6use crate::frame::{self, Frame, Reason};
7use crate::proto::{peer, Error, Initiator, Open, Peer, WindowSize};
8use crate::{client, proto, server};
9
10use bytes::{Buf, Bytes};
11use http::{HeaderMap, Request, Response};
12use std::task::{Context, Poll, Waker};
13use tokio::io::AsyncWrite;
14
15use std::sync::{Arc, Mutex};
16use std::{fmt, io};
17
18#[derive(Debug)]
19pub(crate) struct Streams<B, P>
20where
21 P: Peer,
22{
23 inner: Arc<Mutex<Inner>>,
26
27 send_buffer: Arc<SendBuffer<B>>,
36
37 _p: ::std::marker::PhantomData<P>,
38}
39
40#[derive(Debug)]
43pub(crate) struct DynStreams<'a, B> {
44 inner: &'a Mutex<Inner>,
45
46 send_buffer: &'a SendBuffer<B>,
47
48 peer: peer::Dyn,
49}
50
51#[derive(Debug)]
53pub(crate) struct StreamRef<B> {
54 opaque: OpaqueStreamRef,
55 send_buffer: Arc<SendBuffer<B>>,
56}
57
58pub(crate) struct OpaqueStreamRef {
60 inner: Arc<Mutex<Inner>>,
61 key: store::Key,
62}
63
64#[derive(Debug)]
69struct Inner {
70 counts: Counts,
72
73 actions: Actions,
75
76 store: Store,
78
79 refs: usize,
81}
82
83#[derive(Debug)]
84struct Actions {
85 recv: Recv,
87
88 send: Send,
90
91 task: Option<Waker>,
93
94 conn_error: Option<proto::Error>,
96}
97
98#[derive(Debug)]
100struct SendBuffer<B> {
101 inner: Mutex<Buffer<Frame<B>>>,
102}
103
104impl<B, P> Streams<B, P>
107where
108 B: Buf,
109 P: Peer,
110{
111 pub fn new(config: Config) -> Self {
112 let peer = P::r#dyn();
113
114 Streams {
115 inner: Inner::new(peer, config),
116 send_buffer: Arc::new(SendBuffer::new()),
117 _p: ::std::marker::PhantomData,
118 }
119 }
120
121 pub fn set_target_connection_window_size(&mut self, size: WindowSize) -> Result<(), Reason> {
122 let mut me = self.inner.lock().unwrap();
123 let me = &mut *me;
124
125 me.actions
126 .recv
127 .set_target_connection_window(size, &mut me.actions.task)
128 }
129
130 pub fn next_incoming(&mut self) -> Option<StreamRef<B>> {
131 let mut me = self.inner.lock().unwrap();
132 let me = &mut *me;
133 me.actions.recv.next_incoming(&mut me.store).map(|key| {
134 let stream = &mut me.store.resolve(key);
135 tracing::trace!(
136 "next_incoming; id={:?}, state={:?}",
137 stream.id,
138 stream.state
139 );
140 me.refs += 1;
143
144 if stream.state.is_remote_reset() {
146 me.counts.dec_num_remote_reset_streams();
147 }
148
149 StreamRef {
150 opaque: OpaqueStreamRef::new(self.inner.clone(), stream),
151 send_buffer: self.send_buffer.clone(),
152 }
153 })
154 }
155
156 pub fn send_pending_refusal<T>(
157 &mut self,
158 cx: &mut Context,
159 dst: &mut Codec<T, Prioritized<B>>,
160 ) -> Poll<io::Result<()>>
161 where
162 T: AsyncWrite + Unpin,
163 {
164 let mut me = self.inner.lock().unwrap();
165 let me = &mut *me;
166 me.actions.recv.send_pending_refusal(cx, dst)
167 }
168
169 pub fn clear_expired_reset_streams(&mut self) {
170 let mut me = self.inner.lock().unwrap();
171 let me = &mut *me;
172 me.actions
173 .recv
174 .clear_expired_reset_streams(&mut me.store, &mut me.counts);
175 }
176
177 pub fn poll_complete<T>(
178 &mut self,
179 cx: &mut Context,
180 dst: &mut Codec<T, Prioritized<B>>,
181 ) -> Poll<io::Result<()>>
182 where
183 T: AsyncWrite + Unpin,
184 {
185 let mut me = self.inner.lock().unwrap();
186 me.poll_complete(&self.send_buffer, cx, dst)
187 }
188
189 pub fn apply_remote_settings(
190 &mut self,
191 frame: &frame::Settings,
192 is_initial: bool,
193 ) -> Result<(), Error> {
194 let mut me = self.inner.lock().unwrap();
195 let me = &mut *me;
196
197 let mut send_buffer = self.send_buffer.inner.lock().unwrap();
198 let send_buffer = &mut *send_buffer;
199
200 me.counts.apply_remote_settings(frame, is_initial);
201
202 me.actions.send.apply_remote_settings(
203 frame,
204 send_buffer,
205 &mut me.store,
206 &mut me.counts,
207 &mut me.actions.task,
208 )
209 }
210
211 pub fn apply_local_settings(&mut self, frame: &frame::Settings) -> Result<(), Error> {
212 let mut me = self.inner.lock().unwrap();
213 let me = &mut *me;
214
215 me.actions.recv.apply_local_settings(frame, &mut me.store)
216 }
217
218 pub fn send_request(
219 &mut self,
220 mut request: Request<()>,
221 end_of_stream: bool,
222 pending: Option<&OpaqueStreamRef>,
223 ) -> Result<(StreamRef<B>, bool), SendError> {
224 use super::stream::ContentLength;
225 use http::Method;
226
227 let protocol = request.extensions_mut().remove::<Protocol>();
228
229 request.extensions_mut().clear();
231
232 let mut me = self.inner.lock().unwrap();
238 let me = &mut *me;
239
240 let mut send_buffer = self.send_buffer.inner.lock().unwrap();
241 let send_buffer = &mut *send_buffer;
242
243 me.actions.ensure_no_conn_error()?;
244 me.actions.send.ensure_next_stream_id()?;
245
246 if let Some(stream) = pending {
253 if me.store.resolve(stream.key).is_pending_open {
254 return Err(UserError::Rejected.into());
255 }
256 }
257
258 if me.counts.peer().is_server() {
259 return Err(UserError::UnexpectedFrameType.into());
261 }
262
263 let stream_id = me.actions.send.open()?;
264
265 let mut stream = Stream::new(
266 stream_id,
267 me.actions.send.init_window_sz(),
268 me.actions.recv.init_window_sz(),
269 );
270
271 if *request.method() == Method::HEAD {
272 stream.content_length = ContentLength::Head;
273 }
274
275 let headers =
277 client::Peer::convert_send_message(stream_id, request, protocol, end_of_stream)?;
278
279 let mut stream = me.store.insert(stream.id, stream);
280
281 let sent = me.actions.send.send_headers(
282 headers,
283 send_buffer,
284 &mut stream,
285 &mut me.counts,
286 &mut me.actions.task,
287 );
288
289 if let Err(err) = sent {
292 stream.unlink();
293 stream.remove();
294 return Err(err.into());
295 }
296
297 debug_assert!(!stream.state.is_closed());
300
301 me.refs += 1;
304
305 let is_full = me.counts.next_send_stream_will_reach_capacity();
306 Ok((
307 StreamRef {
308 opaque: OpaqueStreamRef::new(self.inner.clone(), &mut stream),
309 send_buffer: self.send_buffer.clone(),
310 },
311 is_full,
312 ))
313 }
314
315 pub(crate) fn is_extended_connect_protocol_enabled(&self) -> bool {
316 self.inner
317 .lock()
318 .unwrap()
319 .actions
320 .send
321 .is_extended_connect_protocol_enabled()
322 }
323
324 pub fn current_max_send_streams(&self) -> usize {
325 let me = self.inner.lock().unwrap();
326 me.counts.max_send_streams()
327 }
328
329 pub fn current_max_recv_streams(&self) -> usize {
330 let me = self.inner.lock().unwrap();
331 me.counts.max_recv_streams()
332 }
333}
334
335impl<B> DynStreams<'_, B> {
336 pub fn is_buffer_empty(&self) -> bool {
337 self.send_buffer.is_empty()
338 }
339
340 pub fn is_server(&self) -> bool {
341 self.peer.is_server()
342 }
343
344 pub fn recv_headers(&mut self, frame: frame::Headers) -> Result<(), Error> {
345 let mut me = self.inner.lock().unwrap();
346
347 me.recv_headers(self.peer, self.send_buffer, frame)
348 }
349
350 pub fn recv_data(&mut self, frame: frame::Data) -> Result<(), Error> {
351 let mut me = self.inner.lock().unwrap();
352 me.recv_data(self.peer, self.send_buffer, frame)
353 }
354
355 pub fn recv_reset(&mut self, frame: frame::Reset) -> Result<(), Error> {
356 let mut me = self.inner.lock().unwrap();
357
358 me.recv_reset(self.send_buffer, frame)
359 }
360
361 pub fn handle_error(&mut self, err: proto::Error) -> StreamId {
363 let mut me = self.inner.lock().unwrap();
364 me.handle_error(self.send_buffer, err)
365 }
366
367 pub fn recv_go_away(&mut self, frame: &frame::GoAway) -> Result<(), Error> {
368 let mut me = self.inner.lock().unwrap();
369 me.recv_go_away(self.send_buffer, frame)
370 }
371
372 pub fn last_processed_id(&self) -> StreamId {
373 self.inner.lock().unwrap().actions.recv.last_processed_id()
374 }
375
376 pub fn recv_window_update(&mut self, frame: frame::WindowUpdate) -> Result<(), Error> {
377 let mut me = self.inner.lock().unwrap();
378 me.recv_window_update(self.send_buffer, frame)
379 }
380
381 pub fn recv_push_promise(&mut self, frame: frame::PushPromise) -> Result<(), Error> {
382 let mut me = self.inner.lock().unwrap();
383 me.recv_push_promise(self.send_buffer, frame)
384 }
385
386 pub fn recv_eof(&mut self, clear_pending_accept: bool) -> Result<(), ()> {
387 let mut me = self.inner.lock().map_err(|_| ())?;
388 me.recv_eof(self.send_buffer, clear_pending_accept)
389 }
390
391 pub fn send_reset(
392 &mut self,
393 id: StreamId,
394 reason: Reason,
395 ) -> Result<(), crate::proto::error::GoAway> {
396 let mut me = self.inner.lock().unwrap();
397 me.send_reset(self.send_buffer, id, reason)
398 }
399
400 pub fn send_go_away(&mut self, last_processed_id: StreamId) {
401 let mut me = self.inner.lock().unwrap();
402 me.actions.recv.go_away(last_processed_id);
403 }
404}
405
406impl Inner {
407 fn new(peer: peer::Dyn, config: Config) -> Arc<Mutex<Self>> {
408 Arc::new(Mutex::new(Inner {
409 counts: Counts::new(peer, &config),
410 actions: Actions {
411 recv: Recv::new(peer, &config),
412 send: Send::new(&config),
413 task: None,
414 conn_error: None,
415 },
416 store: Store::new(),
417 refs: 1,
418 }))
419 }
420
421 fn recv_headers<B>(
422 &mut self,
423 peer: peer::Dyn,
424 send_buffer: &SendBuffer<B>,
425 frame: frame::Headers,
426 ) -> Result<(), Error> {
427 let id = frame.stream_id();
428
429 if id > self.actions.recv.max_stream_id() {
432 tracing::trace!(
433 "id ({:?}) > max_stream_id ({:?}), ignoring HEADERS",
434 id,
435 self.actions.recv.max_stream_id()
436 );
437 return Ok(());
438 }
439
440 let key = match self.store.find_entry(id) {
441 Entry::Occupied(e) => e.key(),
442 Entry::Vacant(e) => {
443 if !peer.is_server() {
449 if self.actions.may_have_forgotten_stream(peer, id) {
452 tracing::debug!(
453 "recv_headers for old stream={:?}, sending STREAM_CLOSED",
454 id,
455 );
456 return Err(Error::library_reset(id, Reason::STREAM_CLOSED));
457 }
458 }
459
460 match self
461 .actions
462 .recv
463 .open(id, Open::Headers, &mut self.counts)?
464 {
465 Some(stream_id) => {
466 let stream = Stream::new(
467 stream_id,
468 self.actions.send.init_window_sz(),
469 self.actions.recv.init_window_sz(),
470 );
471
472 e.insert(stream)
473 }
474 None => return Ok(()),
475 }
476 }
477 };
478
479 let stream = self.store.resolve(key);
480
481 if stream.state.is_local_error() {
482 tracing::trace!("recv_headers; ignoring trailers on {:?}", stream.id);
486 return Ok(());
487 }
488
489 let actions = &mut self.actions;
490 let mut send_buffer = send_buffer.inner.lock().unwrap();
491 let send_buffer = &mut *send_buffer;
492
493 self.counts.transition(stream, |counts, stream| {
494 tracing::trace!(
495 "recv_headers; stream={:?}; state={:?}",
496 stream.id,
497 stream.state
498 );
499
500 let res = if stream.state.is_recv_headers() {
501 match actions.recv.recv_headers(frame, stream, counts) {
502 Ok(()) => Ok(()),
503 Err(RecvHeaderBlockError::Oversize(resp)) => {
504 if let Some(resp) = resp {
505 let sent = actions.send.send_headers(
506 resp, send_buffer, stream, counts, &mut actions.task);
507 debug_assert!(sent.is_ok(), "oversize response should not fail");
508
509 actions.send.schedule_implicit_reset(
510 stream,
511 Reason::PROTOCOL_ERROR,
512 counts,
513 &mut actions.task);
514
515 actions.recv.enqueue_reset_expiration(stream, counts);
516
517 Ok(())
518 } else {
519 Err(Error::library_reset(stream.id, Reason::PROTOCOL_ERROR))
520 }
521 },
522 Err(RecvHeaderBlockError::State(err)) => Err(err),
523 }
524 } else {
525 if !frame.is_end_stream() {
526 proto_err!(stream: "recv_headers: trailers frame was not EOS; stream={:?}", stream.id);
529 return Err(Error::library_reset(stream.id, Reason::PROTOCOL_ERROR));
530 }
531
532 actions.recv.recv_trailers(frame, stream)
533 };
534
535 actions.reset_on_recv_stream_err(send_buffer, stream, counts, res)
536 })
537 }
538
539 fn recv_data<B>(
540 &mut self,
541 peer: peer::Dyn,
542 send_buffer: &SendBuffer<B>,
543 frame: frame::Data,
544 ) -> Result<(), Error> {
545 let id = frame.stream_id();
546
547 let stream = match self.store.find_mut(&id) {
548 Some(stream) => stream,
549 None => {
550 if id > self.actions.recv.max_stream_id() {
553 tracing::trace!(
554 "id ({:?}) > max_stream_id ({:?}), ignoring DATA",
555 id,
556 self.actions.recv.max_stream_id()
557 );
558 return Ok(());
559 }
560
561 if self.actions.may_have_forgotten_stream(peer, id) {
562 tracing::debug!("recv_data for old stream={:?}, sending STREAM_CLOSED", id,);
563
564 let sz = frame.payload().len();
565 assert!(sz <= super::MAX_WINDOW_SIZE as usize);
568 let sz = sz as WindowSize;
569
570 self.actions.recv.ignore_data(sz)?;
571 return Err(Error::library_reset(id, Reason::STREAM_CLOSED));
572 }
573
574 proto_err!(conn: "recv_data: stream not found; id={:?}", id);
575 return Err(Error::library_go_away(Reason::PROTOCOL_ERROR));
576 }
577 };
578
579 let actions = &mut self.actions;
580 let mut send_buffer = send_buffer.inner.lock().unwrap();
581 let send_buffer = &mut *send_buffer;
582
583 self.counts.transition(stream, |counts, stream| {
584 let sz = frame.payload().len();
585 let res = actions.recv.recv_data(frame, stream);
586
587 if let Err(Error::Reset(..)) = res {
591 actions
592 .recv
593 .release_connection_capacity(sz as WindowSize, &mut None);
594 }
595 actions.reset_on_recv_stream_err(send_buffer, stream, counts, res)
596 })
597 }
598
599 fn recv_reset<B>(
600 &mut self,
601 send_buffer: &SendBuffer<B>,
602 frame: frame::Reset,
603 ) -> Result<(), Error> {
604 let id = frame.stream_id();
605
606 if id.is_zero() {
607 proto_err!(conn: "recv_reset: invalid stream ID 0");
608 return Err(Error::library_go_away(Reason::PROTOCOL_ERROR));
609 }
610
611 if id > self.actions.recv.max_stream_id() {
614 tracing::trace!(
615 "id ({:?}) > max_stream_id ({:?}), ignoring RST_STREAM",
616 id,
617 self.actions.recv.max_stream_id()
618 );
619 return Ok(());
620 }
621
622 let stream = match self.store.find_mut(&id) {
623 Some(stream) => stream,
624 None => {
625 self.actions
627 .ensure_not_idle(self.counts.peer(), id)
628 .map_err(Error::library_go_away)?;
629
630 return Ok(());
631 }
632 };
633
634 let mut send_buffer = send_buffer.inner.lock().unwrap();
635 let send_buffer = &mut *send_buffer;
636
637 let actions = &mut self.actions;
638
639 self.counts.transition(stream, |counts, stream| {
640 actions.recv.recv_reset(frame, stream, counts)?;
641 actions.send.handle_error(send_buffer, stream, counts);
642 assert!(stream.state.is_closed());
643 Ok(())
644 })
645 }
646
647 fn recv_window_update<B>(
648 &mut self,
649 send_buffer: &SendBuffer<B>,
650 frame: frame::WindowUpdate,
651 ) -> Result<(), Error> {
652 let id = frame.stream_id();
653
654 let mut send_buffer = send_buffer.inner.lock().unwrap();
655 let send_buffer = &mut *send_buffer;
656
657 if id.is_zero() {
658 self.actions
659 .send
660 .recv_connection_window_update(frame, &mut self.store, &mut self.counts)
661 .map_err(Error::library_go_away)?;
662 } else {
663 if let Some(mut stream) = self.store.find_mut(&id) {
666 let res = self
667 .actions
668 .send
669 .recv_stream_window_update(
670 frame.size_increment(),
671 send_buffer,
672 &mut stream,
673 &mut self.counts,
674 &mut self.actions.task,
675 )
676 .map_err(|reason| Error::library_reset(id, reason));
677
678 return self.actions.reset_on_recv_stream_err(
679 send_buffer,
680 &mut stream,
681 &mut self.counts,
682 res,
683 );
684 } else {
685 self.actions
686 .ensure_not_idle(self.counts.peer(), id)
687 .map_err(Error::library_go_away)?;
688 }
689 }
690
691 Ok(())
692 }
693
694 fn handle_error<B>(&mut self, send_buffer: &SendBuffer<B>, err: proto::Error) -> StreamId {
695 let actions = &mut self.actions;
696 let counts = &mut self.counts;
697 let mut send_buffer = send_buffer.inner.lock().unwrap();
698 let send_buffer = &mut *send_buffer;
699
700 let last_processed_id = actions.recv.last_processed_id();
701
702 self.store.for_each(|stream| {
703 counts.transition(stream, |counts, stream| {
704 actions.recv.handle_error(&err, &mut *stream);
705 actions.send.handle_error(send_buffer, stream, counts);
706 })
707 });
708
709 actions.conn_error = Some(err);
710
711 last_processed_id
712 }
713
714 fn recv_go_away<B>(
715 &mut self,
716 send_buffer: &SendBuffer<B>,
717 frame: &frame::GoAway,
718 ) -> Result<(), Error> {
719 let actions = &mut self.actions;
720 let counts = &mut self.counts;
721 let mut send_buffer = send_buffer.inner.lock().unwrap();
722 let send_buffer = &mut *send_buffer;
723
724 let last_stream_id = frame.last_stream_id();
725
726 actions.send.recv_go_away(last_stream_id)?;
727
728 let err = Error::remote_go_away(frame.debug_data().clone(), frame.reason());
729
730 self.store.for_each(|stream| {
731 if stream.id > last_stream_id {
732 counts.transition(stream, |counts, stream| {
733 actions.recv.handle_error(&err, &mut *stream);
734 actions.send.handle_error(send_buffer, stream, counts);
735 })
736 }
737 });
738
739 actions.conn_error = Some(err);
740
741 Ok(())
742 }
743
744 fn recv_push_promise<B>(
745 &mut self,
746 send_buffer: &SendBuffer<B>,
747 frame: frame::PushPromise,
748 ) -> Result<(), Error> {
749 let id = frame.stream_id();
750 let promised_id = frame.promised_id();
751
752 let parent_key = match self.store.find_mut(&id) {
754 Some(stream) => {
755 if id > self.actions.recv.max_stream_id() {
758 tracing::trace!(
759 "id ({:?}) > max_stream_id ({:?}), ignoring PUSH_PROMISE",
760 id,
761 self.actions.recv.max_stream_id()
762 );
763 return Ok(());
764 }
765
766 if !stream.state.ensure_recv_open()? {
768 proto_err!(conn: "recv_push_promise: initiating stream is not opened");
769 return Err(Error::library_go_away(Reason::PROTOCOL_ERROR));
770 }
771
772 stream.key()
773 }
774 None => {
775 proto_err!(conn: "recv_push_promise: initiating stream is in an invalid state");
776 return Err(Error::library_go_away(Reason::PROTOCOL_ERROR));
777 }
778 };
779
780 self.actions.recv.ensure_can_reserve()?;
786
787 if self
792 .actions
793 .recv
794 .open(promised_id, Open::PushPromise, &mut self.counts)?
795 .is_none()
796 {
797 return Ok(());
798 }
799
800 let child_key: Option<store::Key> = {
803 let stream = self.store.insert(promised_id, {
805 Stream::new(
806 promised_id,
807 self.actions.send.init_window_sz(),
808 self.actions.recv.init_window_sz(),
809 )
810 });
811
812 let actions = &mut self.actions;
813
814 self.counts.transition(stream, |counts, stream| {
815 let stream_valid = actions.recv.recv_push_promise(frame, stream);
816
817 match stream_valid {
818 Ok(()) => Ok(Some(stream.key())),
819 _ => {
820 let mut send_buffer = send_buffer.inner.lock().unwrap();
821 actions
822 .reset_on_recv_stream_err(
823 &mut *send_buffer,
824 stream,
825 counts,
826 stream_valid,
827 )
828 .map(|()| None)
829 }
830 }
831 })?
832 };
833 if let Some(child) = child_key {
835 let mut ppp = self.store[parent_key].pending_push_promises.take();
836 ppp.push(&mut self.store.resolve(child));
837
838 let parent = &mut self.store.resolve(parent_key);
839 parent.pending_push_promises = ppp;
840 parent.notify_push();
841 };
842
843 Ok(())
844 }
845
846 fn recv_eof<B>(
847 &mut self,
848 send_buffer: &SendBuffer<B>,
849 clear_pending_accept: bool,
850 ) -> Result<(), ()> {
851 let actions = &mut self.actions;
852 let counts = &mut self.counts;
853 let mut send_buffer = send_buffer.inner.lock().unwrap();
854 let send_buffer = &mut *send_buffer;
855
856 if actions.conn_error.is_none() {
857 actions.conn_error = Some(
858 io::Error::new(
859 io::ErrorKind::BrokenPipe,
860 "connection closed because of a broken pipe",
861 )
862 .into(),
863 );
864 }
865
866 tracing::trace!("Streams::recv_eof");
867
868 self.store.for_each(|stream| {
869 counts.transition(stream, |counts, stream| {
870 actions.recv.recv_eof(stream);
871
872 actions.send.handle_error(send_buffer, stream, counts);
875 })
876 });
877
878 actions.clear_queues(clear_pending_accept, &mut self.store, counts);
879 Ok(())
880 }
881
882 fn poll_complete<T, B>(
883 &mut self,
884 send_buffer: &SendBuffer<B>,
885 cx: &mut Context,
886 dst: &mut Codec<T, Prioritized<B>>,
887 ) -> Poll<io::Result<()>>
888 where
889 T: AsyncWrite + Unpin,
890 B: Buf,
891 {
892 let mut send_buffer = send_buffer.inner.lock().unwrap();
893 let send_buffer = &mut *send_buffer;
894
895 ready!(self
900 .actions
901 .recv
902 .poll_complete(cx, &mut self.store, &mut self.counts, dst))?;
903
904 ready!(self.actions.send.poll_complete(
906 cx,
907 send_buffer,
908 &mut self.store,
909 &mut self.counts,
910 dst
911 ))?;
912
913 self.actions.task = Some(cx.waker().clone());
915
916 Poll::Ready(Ok(()))
917 }
918
919 fn send_reset<B>(
920 &mut self,
921 send_buffer: &SendBuffer<B>,
922 id: StreamId,
923 reason: Reason,
924 ) -> Result<(), crate::proto::error::GoAway> {
925 let key = match self.store.find_entry(id) {
926 Entry::Occupied(e) => e.key(),
927 Entry::Vacant(e) => {
928 if self.counts.peer().is_local_init(id) {
941 self.actions.send.maybe_reset_next_stream_id(id);
944 } else {
945 self.actions.recv.maybe_reset_next_stream_id(id);
948 }
949
950 let stream = Stream::new(id, 0, 0);
951
952 e.insert(stream)
953 }
954 };
955
956 let stream = self.store.resolve(key);
957 let mut send_buffer = send_buffer.inner.lock().unwrap();
958 let send_buffer = &mut *send_buffer;
959 self.actions.send_reset(
960 stream,
961 reason,
962 Initiator::Library,
963 &mut self.counts,
964 send_buffer,
965 )
966 }
967}
968
969impl<B> Streams<B, client::Peer>
970where
971 B: Buf,
972{
973 pub fn poll_pending_open(
974 &mut self,
975 cx: &Context,
976 pending: Option<&OpaqueStreamRef>,
977 ) -> Poll<Result<(), crate::Error>> {
978 let mut me = self.inner.lock().unwrap();
979 let me = &mut *me;
980
981 me.actions.ensure_no_conn_error()?;
982 me.actions.send.ensure_next_stream_id()?;
983
984 if let Some(pending) = pending {
985 let mut stream = me.store.resolve(pending.key);
986 tracing::trace!("poll_pending_open; stream = {:?}", stream.is_pending_open);
987 if stream.is_pending_open {
988 stream.wait_send(cx);
989 return Poll::Pending;
990 }
991 }
992 Poll::Ready(Ok(()))
993 }
994}
995
996impl<B, P> Streams<B, P>
997where
998 P: Peer,
999{
1000 pub fn as_dyn(&self) -> DynStreams<'_, B> {
1001 let Self {
1002 inner,
1003 send_buffer,
1004 _p,
1005 } = self;
1006 DynStreams {
1007 inner,
1008 send_buffer,
1009 peer: P::r#dyn(),
1010 }
1011 }
1012
1013 pub fn recv_eof(&mut self, clear_pending_accept: bool) -> Result<(), ()> {
1017 self.as_dyn().recv_eof(clear_pending_accept)
1018 }
1019
1020 pub(crate) fn max_send_streams(&self) -> usize {
1021 self.inner.lock().unwrap().counts.max_send_streams()
1022 }
1023
1024 pub(crate) fn max_recv_streams(&self) -> usize {
1025 self.inner.lock().unwrap().counts.max_recv_streams()
1026 }
1027
1028 #[cfg(feature = "unstable")]
1029 pub fn num_active_streams(&self) -> usize {
1030 let me = self.inner.lock().unwrap();
1031 me.store.num_active_streams()
1032 }
1033
1034 pub fn has_streams(&self) -> bool {
1035 let me = self.inner.lock().unwrap();
1036 me.counts.has_streams()
1037 }
1038
1039 pub fn has_streams_or_other_references(&self) -> bool {
1040 let me = self.inner.lock().unwrap();
1041 me.counts.has_streams() || me.refs > 1
1042 }
1043
1044 #[cfg(feature = "unstable")]
1045 pub fn num_wired_streams(&self) -> usize {
1046 let me = self.inner.lock().unwrap();
1047 me.store.num_wired_streams()
1048 }
1049}
1050
1051impl<B, P> Clone for Streams<B, P>
1053where
1054 P: Peer,
1055{
1056 fn clone(&self) -> Self {
1057 self.inner.lock().unwrap().refs += 1;
1058 Streams {
1059 inner: self.inner.clone(),
1060 send_buffer: self.send_buffer.clone(),
1061 _p: ::std::marker::PhantomData,
1062 }
1063 }
1064}
1065
1066impl<B, P> Drop for Streams<B, P>
1067where
1068 P: Peer,
1069{
1070 fn drop(&mut self) {
1071 if let Ok(mut inner) = self.inner.lock() {
1072 inner.refs -= 1;
1073 if inner.refs == 1 {
1074 if let Some(task) = inner.actions.task.take() {
1075 task.wake();
1076 }
1077 }
1078 }
1079 }
1080}
1081
1082impl<B> StreamRef<B> {
1085 pub fn send_data(&mut self, data: B, end_stream: bool) -> Result<(), UserError>
1086 where
1087 B: Buf,
1088 {
1089 let mut me = self.opaque.inner.lock().unwrap();
1090 let me = &mut *me;
1091
1092 let stream = me.store.resolve(self.opaque.key);
1093 let actions = &mut me.actions;
1094 let mut send_buffer = self.send_buffer.inner.lock().unwrap();
1095 let send_buffer = &mut *send_buffer;
1096
1097 me.counts.transition(stream, |counts, stream| {
1098 let mut frame = frame::Data::new(stream.id, data);
1100 frame.set_end_stream(end_stream);
1101
1102 actions
1104 .send
1105 .send_data(frame, send_buffer, stream, counts, &mut actions.task)
1106 })
1107 }
1108
1109 pub fn send_trailers(&mut self, trailers: HeaderMap) -> Result<(), UserError> {
1110 let mut me = self.opaque.inner.lock().unwrap();
1111 let me = &mut *me;
1112
1113 let stream = me.store.resolve(self.opaque.key);
1114 let actions = &mut me.actions;
1115 let mut send_buffer = self.send_buffer.inner.lock().unwrap();
1116 let send_buffer = &mut *send_buffer;
1117
1118 me.counts.transition(stream, |counts, stream| {
1119 let frame = frame::Headers::trailers(stream.id, trailers);
1121
1122 actions
1124 .send
1125 .send_trailers(frame, send_buffer, stream, counts, &mut actions.task)
1126 })
1127 }
1128
1129 pub fn send_reset(&mut self, reason: Reason) {
1130 let mut me = self.opaque.inner.lock().unwrap();
1131 let me = &mut *me;
1132
1133 let stream = me.store.resolve(self.opaque.key);
1134 let mut send_buffer = self.send_buffer.inner.lock().unwrap();
1135 let send_buffer = &mut *send_buffer;
1136
1137 match me
1138 .actions
1139 .send_reset(stream, reason, Initiator::User, &mut me.counts, send_buffer)
1140 {
1141 Ok(()) => (),
1142 Err(crate::proto::error::GoAway { .. }) => {
1143 unreachable!("Initiator::User should not error sending reset");
1149 }
1150 }
1151 }
1152
1153 pub fn send_informational_headers(&mut self, frame: frame::Headers) -> Result<(), UserError> {
1154 let mut me = self.opaque.inner.lock().unwrap();
1155 let me = &mut *me;
1156
1157 let stream = me.store.resolve(self.opaque.key);
1158 let actions = &mut me.actions;
1159 let mut send_buffer = self.send_buffer.inner.lock().unwrap();
1160 let send_buffer = &mut *send_buffer;
1161
1162 me.counts.transition(stream, |counts, stream| {
1163 debug_assert!(
1169 frame.is_informational(),
1170 "Frame must be informational after conversion from informational response"
1171 );
1172
1173 if frame.is_end_stream() {
1175 return Err(UserError::UnexpectedFrameType);
1176 }
1177
1178 actions.send.send_interim_informational_headers(
1181 frame,
1182 send_buffer,
1183 stream,
1184 counts,
1185 &mut actions.task,
1186 )
1187 })
1188 }
1189
1190 pub fn send_response(
1191 &mut self,
1192 mut response: Response<()>,
1193 end_of_stream: bool,
1194 ) -> Result<(), UserError> {
1195 response.extensions_mut().clear();
1197 let mut me = self.opaque.inner.lock().unwrap();
1198 let me = &mut *me;
1199
1200 let stream = me.store.resolve(self.opaque.key);
1201 let actions = &mut me.actions;
1202 let mut send_buffer = self.send_buffer.inner.lock().unwrap();
1203 let send_buffer = &mut *send_buffer;
1204
1205 me.counts.transition(stream, |counts, stream| {
1206 let frame = server::Peer::convert_send_message(stream.id, response, end_of_stream);
1207
1208 actions
1209 .send
1210 .send_headers(frame, send_buffer, stream, counts, &mut actions.task)
1211 })
1212 }
1213
1214 pub fn send_push_promise(
1215 &mut self,
1216 mut request: Request<()>,
1217 ) -> Result<StreamRef<B>, UserError> {
1218 request.extensions_mut().clear();
1220 let mut me = self.opaque.inner.lock().unwrap();
1221 let me = &mut *me;
1222
1223 let mut send_buffer = self.send_buffer.inner.lock().unwrap();
1224 let send_buffer = &mut *send_buffer;
1225
1226 let actions = &mut me.actions;
1227 let promised_id = actions.send.reserve_local()?;
1228
1229 let child_key = {
1230 let mut child_stream = me.store.insert(
1231 promised_id,
1232 Stream::new(
1233 promised_id,
1234 actions.send.init_window_sz(),
1235 actions.recv.init_window_sz(),
1236 ),
1237 );
1238 child_stream.state.reserve_local()?;
1239 child_stream.is_pending_push = true;
1240 child_stream.key()
1241 };
1242
1243 let pushed = {
1244 let mut stream = me.store.resolve(self.opaque.key);
1245
1246 let frame = crate::server::Peer::convert_push_message(stream.id, promised_id, request)?;
1247
1248 actions
1249 .send
1250 .send_push_promise(frame, send_buffer, &mut stream, &mut actions.task)
1251 };
1252
1253 if let Err(err) = pushed {
1254 let mut child_stream = me.store.resolve(child_key);
1255 child_stream.unlink();
1256 child_stream.remove();
1257 return Err(err);
1258 }
1259
1260 me.refs += 1;
1261 let opaque =
1262 OpaqueStreamRef::new(self.opaque.inner.clone(), &mut me.store.resolve(child_key));
1263
1264 Ok(StreamRef {
1265 opaque,
1266 send_buffer: self.send_buffer.clone(),
1267 })
1268 }
1269
1270 pub fn take_request(&self) -> Request<()> {
1278 let mut me = self.opaque.inner.lock().unwrap();
1279 let me = &mut *me;
1280
1281 let mut stream = me.store.resolve(self.opaque.key);
1282 me.actions.recv.take_request(&mut stream)
1283 }
1284
1285 pub fn is_pending_open(&self) -> bool {
1287 let mut me = self.opaque.inner.lock().unwrap();
1288 me.store.resolve(self.opaque.key).is_pending_open
1289 }
1290
1291 pub fn reserve_capacity(&mut self, capacity: WindowSize) {
1293 let mut me = self.opaque.inner.lock().unwrap();
1294 let me = &mut *me;
1295
1296 let mut stream = me.store.resolve(self.opaque.key);
1297
1298 me.actions
1299 .send
1300 .reserve_capacity(capacity, &mut stream, &mut me.counts)
1301 }
1302
1303 pub fn capacity(&self) -> WindowSize {
1305 let mut me = self.opaque.inner.lock().unwrap();
1306 let me = &mut *me;
1307
1308 let mut stream = me.store.resolve(self.opaque.key);
1309
1310 me.actions.send.capacity(&mut stream)
1311 }
1312
1313 pub fn poll_capacity(&mut self, cx: &Context) -> Poll<Option<Result<WindowSize, UserError>>> {
1315 let mut me = self.opaque.inner.lock().unwrap();
1316 let me = &mut *me;
1317
1318 let mut stream = me.store.resolve(self.opaque.key);
1319
1320 me.actions.send.poll_capacity(cx, &mut stream)
1321 }
1322
1323 pub(crate) fn poll_reset(
1325 &mut self,
1326 cx: &Context,
1327 mode: proto::PollReset,
1328 ) -> Poll<Result<Reason, crate::Error>> {
1329 let mut me = self.opaque.inner.lock().unwrap();
1330 let me = &mut *me;
1331
1332 let mut stream = me.store.resolve(self.opaque.key);
1333
1334 me.actions.send.poll_reset(cx, &mut stream, mode)
1335 }
1336
1337 pub fn clone_to_opaque(&self) -> OpaqueStreamRef {
1338 self.opaque.clone()
1339 }
1340
1341 pub fn stream_id(&self) -> StreamId {
1342 self.opaque.stream_id()
1343 }
1344}
1345
1346impl<B> Clone for StreamRef<B> {
1347 fn clone(&self) -> Self {
1348 StreamRef {
1349 opaque: self.opaque.clone(),
1350 send_buffer: self.send_buffer.clone(),
1351 }
1352 }
1353}
1354
1355impl OpaqueStreamRef {
1358 fn new(inner: Arc<Mutex<Inner>>, stream: &mut store::Ptr) -> OpaqueStreamRef {
1359 stream.ref_inc();
1360 OpaqueStreamRef {
1361 inner,
1362 key: stream.key(),
1363 }
1364 }
1365 pub fn poll_response(&mut self, cx: &Context) -> Poll<Result<Response<()>, proto::Error>> {
1367 let mut me = self.inner.lock().unwrap();
1368 let me = &mut *me;
1369
1370 let mut stream = me.store.resolve(self.key);
1371
1372 me.actions.recv.poll_response(cx, &mut stream)
1373 }
1374
1375 pub fn poll_informational(
1377 &mut self,
1378 cx: &Context,
1379 ) -> Poll<Option<Result<Response<()>, proto::Error>>> {
1380 let mut me = self.inner.lock().unwrap();
1381 let me = &mut *me;
1382
1383 let mut stream = me.store.resolve(self.key);
1384
1385 me.actions.recv.poll_informational(cx, &mut stream)
1386 }
1387 pub fn poll_pushed(
1389 &mut self,
1390 cx: &Context,
1391 ) -> Poll<Option<Result<(Request<()>, OpaqueStreamRef), proto::Error>>> {
1392 let mut me = self.inner.lock().unwrap();
1393 let me = &mut *me;
1394
1395 let mut stream = me.store.resolve(self.key);
1396 me.actions
1397 .recv
1398 .poll_pushed(cx, &mut stream)
1399 .map_ok(|(h, key)| {
1400 me.refs += 1;
1401 let opaque_ref =
1402 OpaqueStreamRef::new(self.inner.clone(), &mut me.store.resolve(key));
1403 (h, opaque_ref)
1404 })
1405 }
1406
1407 pub fn is_end_stream(&self) -> bool {
1408 let mut me = self.inner.lock().unwrap();
1409 let me = &mut *me;
1410
1411 let stream = me.store.resolve(self.key);
1412
1413 me.actions.recv.is_end_stream(&stream)
1414 }
1415
1416 pub fn poll_data(&mut self, cx: &Context) -> Poll<Option<Result<Bytes, proto::Error>>> {
1417 let mut me = self.inner.lock().unwrap();
1418 let me = &mut *me;
1419
1420 let mut stream = me.store.resolve(self.key);
1421
1422 me.actions.recv.poll_data(cx, &mut stream)
1423 }
1424
1425 pub fn poll_trailers(&mut self, cx: &Context) -> Poll<Option<Result<HeaderMap, proto::Error>>> {
1426 let mut me = self.inner.lock().unwrap();
1427 let me = &mut *me;
1428
1429 let mut stream = me.store.resolve(self.key);
1430
1431 me.actions.recv.poll_trailers(cx, &mut stream)
1432 }
1433
1434 pub(crate) fn available_recv_capacity(&self) -> isize {
1435 let me = self.inner.lock().unwrap();
1436 let me = &*me;
1437
1438 let stream = &me.store[self.key];
1439 stream.recv_flow.available().into()
1440 }
1441
1442 pub(crate) fn used_recv_capacity(&self) -> WindowSize {
1443 let me = self.inner.lock().unwrap();
1444 let me = &*me;
1445
1446 let stream = &me.store[self.key];
1447 stream.in_flight_recv_data
1448 }
1449
1450 pub fn release_capacity(&mut self, capacity: WindowSize) -> Result<(), UserError> {
1453 let mut me = self.inner.lock().unwrap();
1454 let me = &mut *me;
1455
1456 let mut stream = me.store.resolve(self.key);
1457
1458 me.actions
1459 .recv
1460 .release_capacity(capacity, &mut stream, &mut me.actions.task)
1461 }
1462
1463 pub(crate) fn clear_recv_buffer(&mut self) {
1465 let mut me = self.inner.lock().unwrap();
1466 let me = &mut *me;
1467
1468 let mut stream = me.store.resolve(self.key);
1469 stream.is_recv = false;
1470 me.actions.recv.clear_recv_buffer(&mut stream);
1471 }
1472
1473 pub fn stream_id(&self) -> StreamId {
1474 self.inner.lock().unwrap().store[self.key].id
1475 }
1476}
1477
1478impl fmt::Debug for OpaqueStreamRef {
1479 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
1480 use std::sync::TryLockError::*;
1481
1482 match self.inner.try_lock() {
1483 Ok(me) => {
1484 let stream = &me.store[self.key];
1485 fmt.debug_struct("OpaqueStreamRef")
1486 .field("stream_id", &stream.id)
1487 .field("ref_count", &stream.ref_count)
1488 .finish()
1489 }
1490 Err(Poisoned(_)) => fmt
1491 .debug_struct("OpaqueStreamRef")
1492 .field("inner", &"<Poisoned>")
1493 .finish(),
1494 Err(WouldBlock) => fmt
1495 .debug_struct("OpaqueStreamRef")
1496 .field("inner", &"<Locked>")
1497 .finish(),
1498 }
1499 }
1500}
1501
1502impl Clone for OpaqueStreamRef {
1503 fn clone(&self) -> Self {
1504 let mut inner = self.inner.lock().unwrap();
1506 inner.store.resolve(self.key).ref_inc();
1507 inner.refs += 1;
1508
1509 OpaqueStreamRef {
1510 inner: self.inner.clone(),
1511 key: self.key,
1512 }
1513 }
1514}
1515
1516impl Drop for OpaqueStreamRef {
1517 fn drop(&mut self) {
1518 drop_stream_ref(&self.inner, self.key);
1519 }
1520}
1521
1522fn drop_stream_ref(inner: &Mutex<Inner>, key: store::Key) {
1524 let mut me = match inner.lock() {
1525 Ok(inner) => inner,
1526 Err(_) => {
1527 if ::std::thread::panicking() {
1528 tracing::trace!("StreamRef::drop; mutex poisoned");
1529 return;
1530 } else {
1531 panic!("StreamRef::drop; mutex poisoned");
1532 }
1533 }
1534 };
1535
1536 let me = &mut *me;
1537 me.refs -= 1;
1538 let mut stream = me.store.resolve(key);
1539
1540 tracing::trace!("drop_stream_ref; stream={:?}", stream);
1541
1542 stream.ref_dec();
1544
1545 let actions = &mut me.actions;
1546
1547 if stream.ref_count == 0 && stream.is_closed() {
1552 if let Some(task) = actions.task.take() {
1553 task.wake();
1554 }
1555 }
1556
1557 me.counts.transition(stream, |counts, stream| {
1558 maybe_cancel(stream, actions, counts);
1559
1560 if stream.ref_count == 0 {
1561 actions
1564 .recv
1565 .release_closed_capacity(stream, &mut actions.task);
1566
1567 let mut ppp = stream.pending_push_promises.take();
1569 while let Some(promise) = ppp.pop(stream.store_mut()) {
1570 counts.transition(promise, |counts, stream| {
1571 maybe_cancel(stream, actions, counts);
1572 });
1573 }
1574 }
1575 });
1576}
1577
1578fn maybe_cancel(stream: &mut store::Ptr, actions: &mut Actions, counts: &mut Counts) {
1579 if stream.is_canceled_interest() {
1580 let reason = if counts.peer().is_server()
1584 && stream.state.is_send_closed()
1585 && stream.state.is_recv_streaming()
1586 {
1587 Reason::NO_ERROR
1588 } else {
1589 Reason::CANCEL
1590 };
1591
1592 actions
1593 .send
1594 .schedule_implicit_reset(stream, reason, counts, &mut actions.task);
1595 actions.recv.enqueue_reset_expiration(stream, counts);
1596 }
1597}
1598
1599impl<B> SendBuffer<B> {
1602 fn new() -> Self {
1603 let inner = Mutex::new(Buffer::new());
1604 SendBuffer { inner }
1605 }
1606
1607 pub fn is_empty(&self) -> bool {
1608 let buf = self.inner.lock().unwrap();
1609 buf.is_empty()
1610 }
1611}
1612
1613impl Actions {
1616 fn send_reset<B>(
1617 &mut self,
1618 stream: store::Ptr,
1619 reason: Reason,
1620 initiator: Initiator,
1621 counts: &mut Counts,
1622 send_buffer: &mut Buffer<Frame<B>>,
1623 ) -> Result<(), crate::proto::error::GoAway> {
1624 counts.transition(stream, |counts, stream| {
1625 if initiator.is_library() {
1626 if counts.can_inc_num_local_error_resets() {
1627 counts.inc_num_local_error_resets();
1628 } else {
1629 tracing::warn!(
1630 "locally-reset streams reached limit ({:?})",
1631 counts.max_local_error_resets().unwrap(),
1632 );
1633 return Err(crate::proto::error::GoAway {
1634 reason: Reason::ENHANCE_YOUR_CALM,
1635 debug_data: "too_many_internal_resets".into(),
1636 });
1637 }
1638 }
1639
1640 self.send.send_reset(
1641 reason,
1642 initiator,
1643 send_buffer,
1644 stream,
1645 counts,
1646 &mut self.task,
1647 );
1648 self.recv.enqueue_reset_expiration(stream, counts);
1649 stream.notify_recv();
1651
1652 Ok(())
1653 })
1654 }
1655
1656 fn reset_on_recv_stream_err<B>(
1657 &mut self,
1658 buffer: &mut Buffer<Frame<B>>,
1659 stream: &mut store::Ptr,
1660 counts: &mut Counts,
1661 res: Result<(), Error>,
1662 ) -> Result<(), Error> {
1663 if let Err(Error::Reset(stream_id, reason, initiator)) = res {
1664 debug_assert_eq!(stream_id, stream.id);
1665
1666 if counts.can_inc_num_local_error_resets() {
1667 counts.inc_num_local_error_resets();
1668
1669 self.send
1671 .send_reset(reason, initiator, buffer, stream, counts, &mut self.task);
1672 self.recv.enqueue_reset_expiration(stream, counts);
1673 stream.notify_recv();
1675 Ok(())
1676 } else {
1677 tracing::warn!(
1678 "reset_on_recv_stream_err; locally-reset streams reached limit ({:?})",
1679 counts.max_local_error_resets().unwrap(),
1680 );
1681 Err(Error::library_go_away_data(
1682 Reason::ENHANCE_YOUR_CALM,
1683 "too_many_internal_resets",
1684 ))
1685 }
1686 } else {
1687 res
1688 }
1689 }
1690
1691 fn ensure_not_idle(&mut self, peer: peer::Dyn, id: StreamId) -> Result<(), Reason> {
1692 if peer.is_local_init(id) {
1693 self.send.ensure_not_idle(id)
1694 } else {
1695 self.recv.ensure_not_idle(id)
1696 }
1697 }
1698
1699 fn ensure_no_conn_error(&self) -> Result<(), proto::Error> {
1700 if let Some(ref err) = self.conn_error {
1701 Err(err.clone())
1702 } else {
1703 Ok(())
1704 }
1705 }
1706
1707 fn may_have_forgotten_stream(&self, peer: peer::Dyn, id: StreamId) -> bool {
1717 if id.is_zero() {
1718 return false;
1719 }
1720 if peer.is_local_init(id) {
1721 self.send.may_have_created_stream(id)
1722 } else {
1723 self.recv.may_have_created_stream(id)
1724 }
1725 }
1726
1727 fn clear_queues(&mut self, clear_pending_accept: bool, store: &mut Store, counts: &mut Counts) {
1728 self.recv.clear_queues(clear_pending_accept, store, counts);
1729 self.send.clear_queues(store, counts);
1730 }
1731}